2dconvs_lstm_doae_256.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:29.196238Z",
     "start_time": "2017-11-17T09:03:28.644004Z"
    },
    "_cell_guid": "679e0d3e-646d-4e96-9eb0-b362d8c6e51f",
    "_uuid": "0d05e5ce89af3e25d1c1fb244d021a1cfa1a058c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import array \n",
    "\n",
    "from pydub import AudioSegment\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Input, Conv2D, Flatten, GlobalMaxPooling2D , GlobalMaxPooling1D, MaxPooling2D, Activation, BatchNormalization, GlobalAveragePooling2D, GlobalMaxPool2D, concatenate, Dense, Dropout\n",
    "from keras.optimizers import Adam\n",
    "from tensorflow.python.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Input, GRU, RepeatVector, BatchNormalization, TimeDistributed, Conv1D\n",
    "from keras.layers import GlobalAveragePooling1D, LSTM, MaxPooling1D\n",
    "from keras import backend as K\n",
    "from keras.layers import  Conv2D, MaxPooling2D, UpSampling2D, Lambda, Reshape\n",
    "import keras\n",
    "from keras.layers import AveragePooling1D\n",
    "from keras.layers import ConvLSTM2D, Bidirectional, CuDNNLSTM, LSTM\n",
    "\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "import matplotlib.pyplot as plt\n",
    "import numba\n",
    "\n",
    "import dask.dataframe as dd\n",
    "from dask.multiprocessing import get\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:29.210749Z",
     "start_time": "2017-11-17T09:03:29.19832Z"
    },
    "_cell_guid": "8ab00801-08b9-44d3-a063-32e82dbf8f58",
    "_uuid": "53c19941676690454dd4b91109976b6c59cb7a40"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "POSSIBLE_LABELS = 'yes no up down left right on off stop go silence unknown'.split()\n",
    "id2name = {i: name for i, name in enumerate(POSSIBLE_LABELS)}\n",
    "name2id = {name: i for i, name in id2name.items()}\n",
    "len(id2name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.7 s, sys: 21.5 s, total: 36.2 s\n",
      "Wall time: 3min 41s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "train_df = pickle.load( open(\"cache/train_df_256_aug.pik\",\"rb\"))\n",
    "valid_df = pickle.load( open(\"cache/valid_df_256.pik\",\"rb\"))\n",
    "silent_df = pickle.load(open(\"cache/silent_df_256.pik\",\"rb\"))\n",
    "unknown_df = pickle.load(open(\"cache/unknown_df_256_aug.pik\",\"rb\"))\n",
    "# test_df =  pickle.load(open(\"cache/test_df_256.pik\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.reset_index(inplace=True)\n",
    "valid_df.reset_index(inplace=True)\n",
    "unknown_df.reset_index(inplace=True)\n",
    "silent_df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = np.load(\"cache/predictions_conv2dlstm_plus_longblend1.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:32.519795Z",
     "start_time": "2017-11-17T09:03:32.483881Z"
    },
    "_cell_guid": "144c6e60-8a83-437d-8b8a-ea065af90923",
    "_uuid": "22e0e6c718171167089fb6df36d3dc43a1029992"
   },
   "outputs": [],
   "source": [
    "#no augmentation since the auto encoder has already seen all the train AND test files \n",
    "\n",
    "def train_generator(train_batch_size):\n",
    "    while True:\n",
    "        \n",
    "        this_train = train_df.groupby('label_id').apply(lambda x: x.sample(n = 2000))\n",
    "        extra_data_size = int(this_train.shape[0]* 0.1)\n",
    "        this_train = pd.concat([silent_df.sample(extra_data_size),\n",
    "                                this_train,\n",
    "                                unknown_df.sample(extra_data_size*2)],axis=0 )\n",
    "        \n",
    "        this_train.reset_index(drop=True,inplace=True)\n",
    "        \n",
    "        shuffled_ids = random.sample(range(this_train.shape[0]), this_train.shape[0])\n",
    "        \n",
    "        for start in range(0, len(shuffled_ids), train_batch_size):\n",
    "            x_batch = []\n",
    "            y_batch = []\n",
    "            \n",
    "            end = min(start + train_batch_size, len(shuffled_ids))\n",
    "            i_train_batch = shuffled_ids[start:end]\n",
    "            for i in i_train_batch:\n",
    "                x_batch.append(this_train.loc[i,'raw'].T)\n",
    "#                 x_batch.append(process_wav_file(this_train.iloc[i], augment=True).T)\n",
    "\n",
    "                y_batch.append(this_train.label_id.values[i])\n",
    "                \n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            y_batch = to_categorical(y_batch, num_classes = len(POSSIBLE_LABELS))\n",
    "            \n",
    "            yield x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 124 ms, sys: 0 ns, total: 124 ms\n",
      "Wall time: 135 ms\n"
     ]
    }
   ],
   "source": [
    "%time t = next(train_generator(256))[0][0].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:32.624289Z",
     "start_time": "2017-11-17T09:03:32.521828Z"
    },
    "_cell_guid": "59a13393-9bc3-4b27-abe2-9c78b3c32ead",
    "_uuid": "6f9a8fbf6e352b1c77b9c22dddf1c5d69382bd5b"
   },
   "outputs": [],
   "source": [
    "def valid_generator(val_batch_size):\n",
    "    while True:\n",
    "        ids = list(range(valid_df.shape[0]))\n",
    "        for start in range(0, len(ids), val_batch_size):\n",
    "            x_batch = []\n",
    "            y_batch = []\n",
    "            end = min(start + val_batch_size, len(ids))\n",
    "            i_val_batch = ids[start:end]\n",
    "            for i in i_val_batch:\n",
    "                x_batch.append(valid_df.loc[i,'raw'].T)\n",
    "                y_batch.append(valid_df.label_id.values[i])\n",
    "\n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            y_batch = to_categorical(y_batch, num_classes = len(POSSIBLE_LABELS))\n",
    "            yield x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_df[\"id\"] = test_df.index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_df.reset_index(drop=True,inplace=True)\n",
    "\n",
    "# test_df.drop([\"level_0\",\"index\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_pseudo_generator(test_batch_size):\n",
    "    this_test = test_df #.sample(int(train_df.shape[0]//5* 0.1))\n",
    "    this_test[\"id\"] = this_test.index.values\n",
    "    \n",
    "#     this_test.reset_index(inplace=True)\n",
    "    while True:\n",
    "\n",
    "        shuffled_ids = random.sample(range(test_df.shape[0]), test_df.shape[0])\n",
    "\n",
    "        for start in range(0, len(test_df), test_batch_size):\n",
    "            x_batch = []\n",
    "            y_batch = []\n",
    "\n",
    "            end = min(start + test_batch_size, len(shuffled_ids))\n",
    "            i_test_batch = shuffled_ids[start:end]\n",
    "\n",
    "            for i in i_test_batch:\n",
    "                x_batch.append(test_df.loc[i,'raw'].T)\n",
    "    #                 x_batch.append(process_wav_file(this_train.iloc[i], augment=True).T)\n",
    "\n",
    "                y_batch.append(test_preds[test_df.loc[i,'id']])\n",
    "\n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            y_batch = np.array(y_batch)\n",
    "\n",
    "            yield x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99975455"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(test_pseudo_generator(14))[1].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MixIterator(object):\n",
    "    def __init__(self, iters):\n",
    "        self.iters = iters\n",
    "        self.multi = type(iters) is list\n",
    "\n",
    "        self.N = 64 \n",
    "\n",
    "    def reset(self):\n",
    "        for it in self.iters: it.reset()\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "\n",
    "    def next(self, *args, **kwargs):\n",
    "\n",
    "        nexts = [next(it) for it in self.iters]\n",
    "#         print nexts[0][0].shape, nexts[1][0].shape\n",
    "        n0 = np.concatenate([n[0] for n in nexts])\n",
    "        n1 = np.concatenate([n[1] for n in nexts])\n",
    "        return (n0, n1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mi = MixIterator([train_generator(50), test_pseudo_generator(14)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# next(mi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11    4221\n",
       "1      270\n",
       "3      264\n",
       "0      261\n",
       "2      260\n",
       "9      260\n",
       "10     257\n",
       "6      257\n",
       "7      256\n",
       "5      256\n",
       "4      247\n",
       "8      246\n",
       "Name: label_id, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df.label_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8 ms, sys: 0 ns, total: 8 ms\n",
      "Wall time: 5.89 ms\n"
     ]
    }
   ],
   "source": [
    "%time t = next(valid_generator(64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(256, 32)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df.loc[0,'raw'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64, 32, 256)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a common practice is to choose a filter size in time which spans 2/3 o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0.3\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_relu(x):\n",
    "    x = BatchNormalization()(x)    \n",
    "    x = Activation('relu')(x)\n",
    "    \n",
    "    return x "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0.\n",
    "\n",
    "def get_conv_stacks( x_in, filter_size=2):\n",
    "    \n",
    "    x = BatchNormalization()(x_in)\n",
    "    \n",
    "    x =  Reshape((timesteps, input_dim,1))(x)\n",
    "    \n",
    "    \n",
    "    \n",
    "    x = Conv2D(32,(7,7),padding='same')(x) #was 32\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    x = Conv2D(64,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    \n",
    "    x = MaxPooling2D((1,3))(x)\n",
    "    \n",
    "    x = Conv2D(64,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv2D(64,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    \n",
    "    \n",
    "    x = MaxPooling2D((2,3))(x)\n",
    "    \n",
    "\n",
    "    x = Conv2D(128,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv2D(128,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    x = MaxPooling2D((1,3))(x)\n",
    "    \n",
    "    \n",
    "    x = Conv2D(128,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv2D(128,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    print int(x.shape[-1]) * int(x.shape[-2])\n",
    "    x = Reshape((16,9*128))(x)\n",
    "    \n",
    "    x = Bidirectional(CuDNNLSTM(128,return_sequences=True))(x)\n",
    "\n",
    "\n",
    "\n",
    "    x = RepeatVector(Dense(128))(x)\n",
    "    embedded = batch_relu(x) \n",
    "    \n",
    "    x = Bidirectional(CuDNNLSTM(128,return_sequences=True))(embedded)\n",
    "    x = Reshape((16,9*128))(x)    \n",
    "    \n",
    "    \n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def embedder(filter_size=2):\n",
    "    v_input = Input(shape=( timesteps,input_dim))\n",
    "    x = BatchNormalization()(v_input)\n",
    "    \n",
    "    x =  Reshape((timesteps, input_dim,1))(x)\n",
    "    \n",
    "    \n",
    "    x = Conv2D(32,(7,7),padding='same')(x) #was 32\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    x = Conv2D(64,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    \n",
    "    x = MaxPooling2D((1,3))(x)\n",
    "    \n",
    "    x = Conv2D(64,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv2D(64,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    \n",
    "    \n",
    "    x = MaxPooling2D((2,3))(x)\n",
    "    \n",
    "\n",
    "    x = Conv2D(128,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv2D(128,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    x = MaxPooling2D((1,3))(x)\n",
    "    \n",
    "    \n",
    "    x = Conv2D(128,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv2D(128,(3,3),padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    print int(x.shape[-1]) * int(x.shape[-2])\n",
    "    x = Reshape((16,9*128))(x)\n",
    "    \n",
    "    x = Bidirectional(CuDNNLSTM(128,return_sequences=False))(x)\n",
    "    \n",
    "    \n",
    "    m = Model(v_input, x)\n",
    "    m.trainable = False        \n",
    "    m.compile(optimizer=Adam(lr=1e-3), loss=\"mean_squared_error\", metrics=['accuracy'])    \n",
    "    return m\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1152\n",
      "1152\n"
     ]
    }
   ],
   "source": [
    "# p = 0.3\n",
    "\n",
    "timesteps, input_dim , latent_dim = 32,256, 128\n",
    "\n",
    "x_logml = Input(shape=(timesteps, input_dim)) #1 channel, 99 time, 161 freqs # S : np.ndarray [shape=(n_mels, t)]\n",
    "\n",
    "x = get_conv_stacks(x_logml)\n",
    "\n",
    "speaker_embedding = embedder(5)\n",
    "speaker_embedding.load_weights(\"weights/speaker_embedding_conv2dlstm.hdf5\")\n",
    "speaker_embedding = speaker_embedding(x_logml)\n",
    "\n",
    "x = concatenate([x,speaker_embedding])\n",
    "x = Dense(256, activation = 'relu')(x) #\n",
    "x = Dropout(0.3)(x)\n",
    "\n",
    "\n",
    "\n",
    "x = Dense(len(POSSIBLE_LABELS), activation = 'softmax', name='targets')(x)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# speaker_embedding.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model = Model(inputs = x_logml, outputs = x)\n",
    "model.compile(optimizer=Adam(lr=1e-3), loss='categorical_crossentropy', metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            (None, 32, 256)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "reshape_5 (Reshape)             (None, 32, 256, 1)   0           input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 32, 256, 32)  1600        reshape_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 32, 256, 32)  128         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 32, 256, 32)  0           batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 32, 256, 64)  18496       activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 32, 256, 64)  256         conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 32, 256, 64)  0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_7 (MaxPooling2D)  (None, 32, 85, 64)   0           activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 32, 85, 64)   36928       max_pooling2d_7[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 32, 85, 64)   256         conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 32, 85, 64)   0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 32, 85, 64)   36928       activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 32, 85, 64)   256         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 32, 85, 64)   0           batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_8 (MaxPooling2D)  (None, 16, 28, 64)   0           activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 16, 28, 128)  73856       max_pooling2d_8[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 16, 28, 128)  512         conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 16, 28, 128)  0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 16, 28, 128)  147584      activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 16, 28, 128)  512         conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 16, 28, 128)  0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_9 (MaxPooling2D)  (None, 16, 9, 128)   0           activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)              (None, 16, 9, 128)   147584      max_pooling2d_9[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 16, 9, 128)   512         conv2d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 16, 9, 128)   0           batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)              (None, 16, 9, 128)   147584      activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, 16, 9, 128)   512         conv2d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 16, 9, 128)   0           batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "reshape_6 (Reshape)             (None, 16, 1152)     0           activation_24[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 256)          1312768     reshape_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "model_2 (Model)                 (None, 256)          1927296     input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 512)          0           bidirectional_3[0][0]            \n",
      "                                                                 model_2[1][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          131328      concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 256)          0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "targets (Dense)                 (None, 12)           3084        dropout_2[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 3,987,980\n",
      "Trainable params: 2,059,212\n",
      "Non-trainable params: 1,928,768\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import TensorBoard\n",
    "exp_name = \"conv2d_wlstm_w2dconv_speaker_embedding\" #max_freqconvs_2510_avgshortcuts\n",
    "callbacks = [EarlyStopping(monitor='val_loss',\n",
    "                           patience=5,\n",
    "                           verbose=1),\n",
    "             \n",
    "             ReduceLROnPlateau(monitor='val_loss',\n",
    "                               factor=0.1,\n",
    "                               patience=3,\n",
    "                               verbose=1,\n",
    "                               epsilon=0.01,\n",
    "                              min_lr=1e-6),\n",
    "             ModelCheckpoint(monitor='val_loss',\n",
    "                             filepath='weights/{}.hdf5'.format(exp_name),\n",
    "                             save_best_only=True,\n",
    "                             save_weights_only=True,\n",
    "                             mode='min')\n",
    "#              , TensorBoard(log_dir='./logs/logs_{}'.format(exp_name), histogram_freq=0, batch_size=64, write_graph=True)\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('./weights/conv2d_wlstm.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:42:31.48233Z",
     "start_time": "2017-11-17T09:03:33.355603Z"
    },
    "_cell_guid": "5f3d1b09-500f-410e-820a-8eaab24b6ebb",
    "_uuid": "528ec66a0a6caca952273ab916e609625839b19e",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "329/329 [==============================] - 147s 446ms/step - loss: 1.2665 - acc: 0.5675 - val_loss: 0.7525 - val_acc: 0.7494\n",
      "Epoch 2/100\n",
      "329/329 [==============================] - 144s 436ms/step - loss: 0.4858 - acc: 0.8466 - val_loss: 0.4204 - val_acc: 0.8594\n",
      "Epoch 3/100\n",
      "329/329 [==============================] - 143s 434ms/step - loss: 0.3577 - acc: 0.8854 - val_loss: 0.2978 - val_acc: 0.9039\n",
      "Epoch 4/100\n",
      "329/329 [==============================] - 144s 436ms/step - loss: 0.3145 - acc: 0.8992 - val_loss: 0.2452 - val_acc: 0.9210\n",
      "Epoch 5/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.2820 - acc: 0.9090 - val_loss: 0.2308 - val_acc: 0.9253\n",
      "Epoch 6/100\n",
      "329/329 [==============================] - 144s 438ms/step - loss: 0.2512 - acc: 0.9198 - val_loss: 0.2010 - val_acc: 0.9392\n",
      "Epoch 7/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.2419 - acc: 0.9212 - val_loss: 0.2076 - val_acc: 0.9365\n",
      "Epoch 8/100\n",
      "329/329 [==============================] - 143s 435ms/step - loss: 0.2206 - acc: 0.9281 - val_loss: 0.2029 - val_acc: 0.9431\n",
      "Epoch 9/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.2060 - acc: 0.9332 - val_loss: 0.1840 - val_acc: 0.9418\n",
      "Epoch 10/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.2002 - acc: 0.9356 - val_loss: 0.1747 - val_acc: 0.9456\n",
      "Epoch 11/100\n",
      "329/329 [==============================] - 144s 438ms/step - loss: 0.1933 - acc: 0.9357 - val_loss: 0.1570 - val_acc: 0.9494\n",
      "Epoch 12/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.1898 - acc: 0.9381 - val_loss: 0.1796 - val_acc: 0.9418\n",
      "Epoch 13/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.1763 - acc: 0.9436 - val_loss: 0.1417 - val_acc: 0.9562\n",
      "Epoch 14/100\n",
      "329/329 [==============================] - 143s 436ms/step - loss: 0.1687 - acc: 0.9442 - val_loss: 0.1713 - val_acc: 0.9436\n",
      "Epoch 15/100\n",
      "329/329 [==============================] - 143s 436ms/step - loss: 0.1641 - acc: 0.9472 - val_loss: 0.1563 - val_acc: 0.9499\n",
      "Epoch 16/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.1500 - acc: 0.9504 - val_loss: 0.1343 - val_acc: 0.9595\n",
      "Epoch 17/100\n",
      "328/329 [============================>.] - ETA: 0s - loss: 0.1487 - acc: 0.9524\n",
      "Epoch 00017: reducing learning rate to 0.00010000000475.\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.1488 - acc: 0.9523 - val_loss: 0.1721 - val_acc: 0.9485\n",
      "Epoch 18/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.1212 - acc: 0.9602 - val_loss: 0.1108 - val_acc: 0.9662\n",
      "Epoch 19/100\n",
      "329/329 [==============================] - 144s 437ms/step - loss: 0.0981 - acc: 0.9682 - val_loss: 0.1096 - val_acc: 0.9672\n",
      "Epoch 20/100\n",
      "329/329 [==============================] - 144s 436ms/step - loss: 0.0962 - acc: 0.9689 - val_loss: 0.1139 - val_acc: 0.9678\n",
      "Epoch 21/100\n",
      "329/329 [==============================] - 142s 433ms/step - loss: 0.0987 - acc: 0.9696 - val_loss: 0.1038 - val_acc: 0.9692\n",
      "Epoch 22/100\n",
      "328/329 [============================>.] - ETA: 0s - loss: 0.0873 - acc: 0.9721\n",
      "Epoch 00022: reducing learning rate to 1.0000000475e-05.\n",
      "329/329 [==============================] - 142s 432ms/step - loss: 0.0874 - acc: 0.9721 - val_loss: 0.1125 - val_acc: 0.9664\n",
      "Epoch 23/100\n",
      "329/329 [==============================] - 142s 432ms/step - loss: 0.0815 - acc: 0.9747 - val_loss: 0.1076 - val_acc: 0.9685\n",
      "Epoch 24/100\n",
      "329/329 [==============================] - 142s 432ms/step - loss: 0.0851 - acc: 0.9727 - val_loss: 0.1048 - val_acc: 0.9691\n",
      "Epoch 25/100\n",
      "328/329 [============================>.] - ETA: 0s - loss: 0.0889 - acc: 0.9724\n",
      "Epoch 00025: reducing learning rate to 1.00000006569e-06.\n",
      "329/329 [==============================] - 142s 432ms/step - loss: 0.0886 - acc: 0.9725 - val_loss: 0.1008 - val_acc: 0.9710\n",
      "Epoch 26/100\n",
      "329/329 [==============================] - 142s 431ms/step - loss: 0.0798 - acc: 0.9735 - val_loss: 0.1023 - val_acc: 0.9705\n",
      "Epoch 27/100\n",
      "329/329 [==============================] - 142s 433ms/step - loss: 0.0789 - acc: 0.9737 - val_loss: 0.1001 - val_acc: 0.9707\n",
      "Epoch 28/100\n",
      "329/329 [==============================] - 142s 432ms/step - loss: 0.0817 - acc: 0.9746 - val_loss: 0.1028 - val_acc: 0.9701\n",
      "Epoch 29/100\n",
      "329/329 [==============================] - 142s 432ms/step - loss: 0.0747 - acc: 0.9762 - val_loss: 0.1030 - val_acc: 0.9698\n",
      "Epoch 30/100\n",
      "329/329 [==============================] - 142s 431ms/step - loss: 0.0802 - acc: 0.9746 - val_loss: 0.1037 - val_acc: 0.9697\n",
      "Epoch 31/100\n",
      "329/329 [==============================] - 142s 431ms/step - loss: 0.0867 - acc: 0.9718 - val_loss: 0.1034 - val_acc: 0.9697\n",
      "Epoch 32/100\n",
      " 58/329 [====>.........................] - ETA: 1:40 - loss: 0.0697 - acc: 0.9774"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-5d96ac088973>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m                               \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                               \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m                               validation_steps=int(np.ceil(valid_df.shape[0]/batch_size)))\n\u001b[0m",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/legacy/interfaces.pyc\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   2145\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   2146\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2147\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   2148\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2149\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1837\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1838\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1839\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1840\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1841\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2355\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2356\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2357\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2358\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1121\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1321\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1322\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1300\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1302\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1304\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "\n",
    "history = model.fit_generator(generator=train_generator(batch_size),\n",
    "                              steps_per_epoch=train_df.shape[0]*(1./5)//batch_size,\n",
    "                              epochs=100,\n",
    "                              callbacks=callbacks,\n",
    "                              validation_data=valid_generator(batch_size),\n",
    "                              validation_steps=int(np.ceil(valid_df.shape[0]/batch_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('./weights/{}.hdf5'.format(exp_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "mi = MixIterator([train_generator(45), valid_generator(5),test_pseudo_generator(14)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "329/329 [==============================] - 106s 324ms/step - loss: 0.1461 - acc: 0.9745 - val_loss: 0.0898 - val_acc: 0.9730\n",
      "Epoch 2/5\n",
      "329/329 [==============================] - 106s 322ms/step - loss: 0.1472 - acc: 0.9762 - val_loss: 0.0912 - val_acc: 0.9727\n",
      "Epoch 3/5\n",
      "329/329 [==============================] - 105s 320ms/step - loss: 0.1496 - acc: 0.9748 - val_loss: 0.0892 - val_acc: 0.9734\n",
      "Epoch 4/5\n",
      "329/329 [==============================] - 105s 320ms/step - loss: 0.1508 - acc: 0.9741 - val_loss: 0.0863 - val_acc: 0.9744\n",
      "Epoch 5/5\n",
      "329/329 [==============================] - 105s 320ms/step - loss: 0.1486 - acc: 0.9756 - val_loss: 0.0869 - val_acc: 0.9745\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "\n",
    "history = model.fit_generator(generator=mi,#train_generator(batch_size),\n",
    "                              steps_per_epoch=train_df.shape[0]*(1./5)//batch_size,\n",
    "                              epochs=5,\n",
    "                              callbacks=callbacks,\n",
    "                              validation_data=valid_generator(batch_size),\n",
    "                              validation_steps=int(np.ceil(valid_df.shape[0]/batch_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "329/329 [==============================] - 105s 318ms/step - loss: 0.1481 - acc: 0.9758 - val_loss: 0.0857 - val_acc: 0.9747\n",
      "Epoch 2/10\n",
      "329/329 [==============================] - 105s 318ms/step - loss: 0.1511 - acc: 0.9750 - val_loss: 0.0871 - val_acc: 0.9744\n",
      "Epoch 3/10\n",
      "329/329 [==============================] - 105s 319ms/step - loss: 0.1511 - acc: 0.9750 - val_loss: 0.0853 - val_acc: 0.9748\n",
      "Epoch 4/10\n",
      "329/329 [==============================] - 105s 319ms/step - loss: 0.1490 - acc: 0.9762 - val_loss: 0.0834 - val_acc: 0.9757\n",
      "Epoch 5/10\n",
      "328/329 [============================>.] - ETA: 0s - loss: 0.1488 - acc: 0.9748\n",
      "Epoch 00005: reducing learning rate to 1e-06.\n",
      "329/329 [==============================] - 105s 320ms/step - loss: 0.1488 - acc: 0.9749 - val_loss: 0.0843 - val_acc: 0.9753\n",
      "Epoch 6/10\n",
      "329/329 [==============================] - 105s 319ms/step - loss: 0.1491 - acc: 0.9745 - val_loss: 0.0841 - val_acc: 0.9748\n",
      "Epoch 7/10\n",
      "329/329 [==============================] - 105s 319ms/step - loss: 0.1522 - acc: 0.9748 - val_loss: 0.0846 - val_acc: 0.9748\n",
      "Epoch 8/10\n",
      "329/329 [==============================] - 105s 319ms/step - loss: 0.1432 - acc: 0.9758 - val_loss: 0.0835 - val_acc: 0.9754\n",
      "Epoch 9/10\n",
      "329/329 [==============================] - 105s 320ms/step - loss: 0.1495 - acc: 0.9756 - val_loss: 0.0832 - val_acc: 0.9751\n",
      "Epoch 10/10\n",
      "329/329 [==============================] - 105s 320ms/step - loss: 0.1532 - acc: 0.9735 - val_loss: 0.0827 - val_acc: 0.9753\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "\n",
    "history = model.fit_generator(generator=mi,#train_generator(batch_size),\n",
    "                              steps_per_epoch=train_df.shape[0]*(1./5)//batch_size,\n",
    "                              epochs=10,\n",
    "                              callbacks=callbacks,\n",
    "                              validation_data=valid_generator(batch_size),\n",
    "                              validation_steps=int(np.ceil(valid_df.shape[0]/batch_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### valid evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_preds = model.predict_generator(valid_generator(64),steps=int(np.ceil(valid_df.shape[0]/64.)))\n",
    "val_preds = np.argmax(val_preds,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7055,)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "oe = OneHotEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = oe.fit_transform(valid_df.label_id.values.reshape(-1, 1)).todense()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_preds = oe.transform(val_preds.reshape(-1, 1)).todense()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.97      0.98       261\n",
      "         no       0.91      0.97      0.94       270\n",
      "         up       0.92      0.95      0.93       260\n",
      "       down       0.98      0.96      0.97       264\n",
      "       left       0.97      0.99      0.98       247\n",
      "      right       0.98      0.98      0.98       256\n",
      "         on       0.95      0.95      0.95       257\n",
      "        off       0.94      0.96      0.95       256\n",
      "       stop       0.96      0.96      0.96       246\n",
      "         go       0.93      0.93      0.93       260\n",
      "    silence       0.96      1.00      0.98       257\n",
      "    unknown       0.99      0.98      0.99      4221\n",
      "\n",
      "avg / total       0.98      0.98      0.98      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.97      0.98       261\n",
      "         no       0.91      0.97      0.94       270\n",
      "         up       0.92      0.94      0.93       260\n",
      "       down       0.98      0.96      0.97       264\n",
      "       left       0.97      0.99      0.98       247\n",
      "      right       0.98      0.98      0.98       256\n",
      "         on       0.95      0.95      0.95       257\n",
      "        off       0.93      0.96      0.94       256\n",
      "       stop       0.96      0.96      0.96       246\n",
      "         go       0.93      0.92      0.92       260\n",
      "    silence       0.96      1.00      0.98       257\n",
      "    unknown       0.99      0.98      0.99      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.97      0.98       261\n",
      "         no       0.91      0.96      0.93       270\n",
      "         up       0.92      0.94      0.93       260\n",
      "       down       0.97      0.96      0.97       264\n",
      "       left       0.97      0.99      0.98       247\n",
      "      right       0.97      0.98      0.98       256\n",
      "         on       0.94      0.95      0.94       257\n",
      "        off       0.92      0.96      0.94       256\n",
      "       stop       0.96      0.96      0.96       246\n",
      "         go       0.92      0.92      0.92       260\n",
      "    silence       0.95      1.00      0.98       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.97      0.97       261\n",
      "         no       0.91      0.95      0.93       270\n",
      "         up       0.93      0.94      0.94       260\n",
      "       down       0.96      0.97      0.96       264\n",
      "       left       0.95      0.99      0.97       247\n",
      "      right       0.95      0.97      0.96       256\n",
      "         on       0.95      0.95      0.95       257\n",
      "        off       0.92      0.96      0.94       256\n",
      "       stop       0.95      0.96      0.95       246\n",
      "         go       0.92      0.92      0.92       260\n",
      "    silence       0.96      1.00      0.98       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tta test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def augment_wav(wav,pval=0.5):\n",
    "    sample_rate = 16000\n",
    "    L = 1000 #16000  # 1 sec\n",
    "    \n",
    "#     adjust speed, with 50% chance\n",
    "    wav = speed_change(wav,1.+ random.choice([.1,-0.1,0])) #random.uniform(-1, 1)*0.05) if np.random.random() < pval else wav\n",
    "    \n",
    "    \n",
    "    #adjust volume\n",
    "#     db_adjustment = random.uniform(-1, 1)*10\n",
    "    wav = wav + random.choice([-10,-5,0,5,10]) #randodb_adjustment if np.random.random() < pval else wav\n",
    "     \n",
    "        \n",
    "    #fill to 1 second\n",
    "    wav = fill_to_1sec(wav)        \n",
    "        \n",
    "    #shift the audio by 10 ms\n",
    "    shift_length = 100\n",
    "    if np.random.random() < 0.5: #shift to left\n",
    "        wav = wav[:L-shift_length]+ AudioSegment.silent(shift_length,frame_rate=sample_rate)\n",
    "    else: #shift to right\n",
    "        wav = AudioSegment.silent(shift_length,frame_rate=sample_rate) + wav[shift_length:]\n",
    "        \n",
    "        \n",
    "        \n",
    "    #blend original file with background noise     \n",
    "#     if np.random.random() < pval:\n",
    "    noise = random.choice(silence_files_AS)\n",
    "    db_delta = (wav.dBFS - noise.dBFS) -10.\n",
    "\n",
    "    if db_delta< 0: #reduce intensity of loud background; if it's too silent, leave it be\n",
    "        noise = noise  + db_delta\n",
    "    wav = wav.overlay(noise)\n",
    " \n",
    "    return wav\n",
    "\n",
    "\n",
    "def process_wav_file(record, reshape=False, augment=False,pval=0.5 ,output_format='logmel',n_mels=128 ):\n",
    "    \n",
    "    if type(record) == str: # test files\n",
    "        fname = record\n",
    "        label = \"test\"\n",
    "    else:    \n",
    "        fname  = record.wav_file\n",
    "        label = record.label\n",
    "\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "    if \"raw_AS_wav\" in record: \n",
    "        wav = record.raw_AS_wav\n",
    "    else:\n",
    "        wav = AudioSegment.from_wav(fname.replace(\"\\\\\",\"/\"))\n",
    "        \n",
    "        \n",
    "    \n",
    "    if (not label in [\"silence\"]) and augment: #no augmentation for sample files \n",
    "        wav = augment_wav(wav,pval)\n",
    "\n",
    "    else: #make sure segment is 1 second\n",
    "        wav = fill_to_1sec(wav)\n",
    "\n",
    "        \n",
    "    samples = AS_to_raw(wav)\n",
    "    \n",
    "    \n",
    "    \n",
    "    if output_format == \"logmel\":\n",
    "        output = log_mel(samples,reshape=reshape,n_mels=n_mels)\n",
    "        \n",
    "    elif output_format == \"mfcc\":\n",
    "        log_S = log_mel(samples,reshape=False,n_mels=n_mels)\n",
    "        mfcc = librosa.feature.mfcc(S=log_S, n_mfcc=40) #hirese mfcc\n",
    "        delta1 = librosa.feature.delta(mfcc, order=1)#hirese mfcc\n",
    "        delta2 = librosa.feature.delta(mfcc, order=2)\n",
    "\n",
    "        output = np.stack([mfcc,delta1,delta2])\n",
    "        \n",
    "    elif  output_format == \"cqt\":   \n",
    "        output = librosa.cqt(samples, sr=16000)\n",
    "    else:\n",
    "        output = samples\n",
    "    \n",
    "    \n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "silence_files_AS = [AudioSegment.from_wav(x) for x in silent_df.wav_file.values]\n",
    "\n",
    "filler = AudioSegment.silent(duration=1000, frame_rate = 16000)\n",
    "\n",
    "def valid_generator_aug(val_batch_size):\n",
    "    while True:\n",
    "        ids = list(range(valid_df.shape[0]))\n",
    "        for start in range(0, len(ids), val_batch_size):\n",
    "            x_batch = []\n",
    "            y_batch = []\n",
    "            end = min(start + val_batch_size, len(ids))\n",
    "            i_val_batch = ids[start:end]\n",
    "            for i in i_val_batch:\n",
    "                x_batch.append(process_wav_file(valid_df.loc[i],n_mels=256,augment=True).T)\n",
    "                y_batch.append(valid_df.label_id.values[i])\n",
    "\n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            y_batch = to_categorical(y_batch, num_classes = len(POSSIBLE_LABELS))\n",
    "            yield x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "111/111 [==============================] - 77s 690ms/step\n",
      "111/111 [==============================] - 77s 698ms/step\n"
     ]
    }
   ],
   "source": [
    "augs= []\n",
    "num_augs = 2\n",
    "for i in range(num_augs):\n",
    "    tmp_gen = valid_generator_aug(64)\n",
    "    augs.append(model.predict_generator(tmp_gen,steps=int(np.ceil(valid_df.shape[0]/64.)),verbose=1))\n",
    "    del tmp_gen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "augs.append(val_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_blend = np.mean(augs,axis=0) #np.mean([val_preds,preds_aug1],axis=0)\n",
    "val_blend = np.argmax(val_blend,axis=1)\n",
    "val_blend = oe.transform(val_blend.reshape(-1, 1)).todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.97      0.98       261\n",
      "         no       0.90      0.94      0.92       270\n",
      "         up       0.92      0.95      0.94       260\n",
      "       down       0.94      0.96      0.95       264\n",
      "       left       0.95      0.98      0.97       247\n",
      "      right       0.94      0.98      0.96       256\n",
      "         on       0.94      0.94      0.94       257\n",
      "        off       0.92      0.96      0.94       256\n",
      "       stop       0.95      0.96      0.95       246\n",
      "         go       0.89      0.90      0.90       260\n",
      "    silence       0.96      1.00      0.98       257\n",
      "    unknown       0.99      0.97      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_blend,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.97      0.97       261\n",
      "         no       0.92      0.94      0.93       270\n",
      "         up       0.89      0.93      0.91       260\n",
      "       down       0.96      0.97      0.96       264\n",
      "       left       0.95      0.99      0.97       247\n",
      "      right       0.96      0.97      0.97       256\n",
      "         on       0.91      0.95      0.93       257\n",
      "        off       0.92      0.95      0.93       256\n",
      "       stop       0.97      0.95      0.96       246\n",
      "         go       0.93      0.93      0.93       260\n",
      "    silence       0.99      1.00      0.99       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.96      0.97       261\n",
      "         no       0.91      0.94      0.92       270\n",
      "         up       0.93      0.92      0.92       260\n",
      "       down       0.93      0.97      0.95       264\n",
      "       left       0.95      0.98      0.96       247\n",
      "      right       0.95      0.95      0.95       256\n",
      "         on       0.93      0.93      0.93       257\n",
      "        off       0.87      0.95      0.91       256\n",
      "       stop       0.96      0.94      0.95       246\n",
      "         go       0.90      0.91      0.90       260\n",
      "    silence       0.99      1.00      0.99       257\n",
      "    unknown       0.98      0.97      0.98      4221\n",
      "\n",
      "avg / total       0.96      0.96      0.96      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.97      0.98       261\n",
      "         no       0.91      0.93      0.92       270\n",
      "         up       0.91      0.93      0.92       260\n",
      "       down       0.97      0.95      0.96       264\n",
      "       left       0.97      0.98      0.97       247\n",
      "      right       0.97      0.94      0.96       256\n",
      "         on       0.95      0.94      0.95       257\n",
      "        off       0.94      0.96      0.95       256\n",
      "       stop       0.97      0.91      0.94       246\n",
      "         go       0.88      0.87      0.87       260\n",
      "    silence       0.99      1.00      1.00       257\n",
      "    unknown       0.98      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.96      0.97       261\n",
      "         no       0.92      0.93      0.92       270\n",
      "         up       0.83      0.95      0.89       260\n",
      "       down       0.95      0.96      0.96       264\n",
      "       left       0.95      0.98      0.97       247\n",
      "      right       0.97      0.95      0.96       256\n",
      "         on       0.94      0.94      0.94       257\n",
      "        off       0.90      0.95      0.92       256\n",
      "       stop       0.96      0.94      0.95       246\n",
      "         go       0.92      0.92      0.92       260\n",
      "    silence       1.00      1.00      1.00       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.97      0.98       261\n",
      "         no       0.89      0.92      0.90       270\n",
      "         up       0.92      0.94      0.93       260\n",
      "       down       0.96      0.97      0.96       264\n",
      "       left       0.96      0.98      0.97       247\n",
      "      right       0.97      0.97      0.97       256\n",
      "         on       0.93      0.95      0.94       257\n",
      "        off       0.92      0.97      0.94       256\n",
      "       stop       0.97      0.94      0.96       246\n",
      "         go       0.91      0.91      0.91       260\n",
      "    silence       0.99      1.00      1.00       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.96      0.97       261\n",
      "         no       0.88      0.91      0.89       270\n",
      "         up       0.86      0.95      0.90       260\n",
      "       down       0.92      0.95      0.93       264\n",
      "       left       0.94      0.98      0.96       247\n",
      "      right       0.94      0.95      0.95       256\n",
      "         on       0.83      0.94      0.88       257\n",
      "        off       0.87      0.94      0.90       256\n",
      "       stop       0.98      0.93      0.95       246\n",
      "         go       0.80      0.90      0.85       260\n",
      "    silence       0.95      1.00      0.98       257\n",
      "    unknown       0.98      0.95      0.97      4221\n",
      "\n",
      "avg / total       0.95      0.95      0.95      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.96      0.97       261\n",
      "         no       0.92      0.92      0.92       270\n",
      "         up       0.93      0.93      0.93       260\n",
      "       down       0.99      0.95      0.97       264\n",
      "       left       0.98      0.97      0.97       247\n",
      "      right       0.99      0.95      0.97       256\n",
      "         on       0.97      0.92      0.95       257\n",
      "        off       0.94      0.92      0.93       256\n",
      "       stop       0.99      0.94      0.97       246\n",
      "         go       0.95      0.88      0.92       260\n",
      "    silence       0.97      1.00      0.98       257\n",
      "    unknown       0.76      0.98      0.86       257\n",
      "\n",
      "avg / total       0.95      0.94      0.94      3091\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.96      0.97       261\n",
      "         no       0.92      0.94      0.93       270\n",
      "         up       0.92      0.93      0.93       260\n",
      "       down       0.98      0.94      0.96       264\n",
      "       left       0.97      0.97      0.97       247\n",
      "      right       0.95      0.96      0.96       256\n",
      "         on       0.96      0.95      0.96       257\n",
      "        off       0.94      0.92      0.93       256\n",
      "       stop       1.00      0.94      0.97       246\n",
      "         go       0.96      0.89      0.92       260\n",
      "    silence       0.98      1.00      0.99       257\n",
      "    unknown       0.82      0.95      0.88       257\n",
      "\n",
      "avg / total       0.95      0.95      0.95      3091\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.95      0.97       261\n",
      "         no       0.94      0.95      0.95       270\n",
      "         up       0.92      0.95      0.93       260\n",
      "       down       0.99      0.95      0.97       264\n",
      "       left       0.97      0.96      0.96       247\n",
      "      right       0.99      0.95      0.97       256\n",
      "         on       0.99      0.92      0.95       257\n",
      "        off       0.93      0.94      0.93       256\n",
      "       stop       0.98      0.95      0.97       246\n",
      "         go       0.94      0.90      0.92       260\n",
      "    silence       0.99      1.00      1.00       257\n",
      "    unknown       0.80      0.96      0.88       257\n",
      "\n",
      "avg / total       0.95      0.95      0.95      3091\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       1.00      0.96      0.98       261\n",
      "         no       0.91      0.94      0.93       270\n",
      "         up       0.90      0.95      0.92       260\n",
      "       down       0.99      0.94      0.96       264\n",
      "       left       0.98      0.98      0.98       247\n",
      "      right       0.98      0.96      0.97       256\n",
      "         on       0.98      0.93      0.95       257\n",
      "        off       0.93      0.93      0.93       256\n",
      "       stop       0.96      0.93      0.95       246\n",
      "         go       0.95      0.90      0.92       260\n",
      "    silence       0.98      1.00      0.99       257\n",
      "    unknown       0.82      0.94      0.87       257\n",
      "\n",
      "avg / total       0.95      0.95      0.95      3091\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T10:28:14.451612Z",
     "start_time": "2017-11-17T10:28:13.307142Z"
    },
    "_cell_guid": "72f27090-c0d1-4d0b-8027-34c915429a79",
    "_uuid": "1007977fccadecdae582ec5d8d52dd3c4c3010aa"
   },
   "outputs": [],
   "source": [
    "test_paths = glob(os.path.join('./data/', 'test/audio/*wav'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "158538"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pickle.load( open(\"cache/test_df_256.pik\",\"rb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158538, 5)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>label_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>wav_file</th>\n",
       "      <th>raw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test</td>\n",
       "      <td>-1</td>\n",
       "      <td>test</td>\n",
       "      <td>./data/test/audio/clip_bd6d0fb25.wav</td>\n",
       "      <td>[[-57.2323073539, -62.8251292953, -75.84322783...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test</td>\n",
       "      <td>-1</td>\n",
       "      <td>test</td>\n",
       "      <td>./data/test/audio/clip_3e7a56353.wav</td>\n",
       "      <td>[[-58.4158758303, -52.5011356178, -53.86649120...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test</td>\n",
       "      <td>-1</td>\n",
       "      <td>test</td>\n",
       "      <td>./data/test/audio/clip_c5884a6cb.wav</td>\n",
       "      <td>[[-73.8001558488, -76.330194253, -72.386112091...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>test</td>\n",
       "      <td>-1</td>\n",
       "      <td>test</td>\n",
       "      <td>./data/test/audio/clip_ebf0d1f7b.wav</td>\n",
       "      <td>[[-71.7484306025, -64.9348066752, -62.39292603...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test</td>\n",
       "      <td>-1</td>\n",
       "      <td>test</td>\n",
       "      <td>./data/test/audio/clip_2f714f052.wav</td>\n",
       "      <td>[[-49.6434011746, -49.9060486303, -49.58891626...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label  label_id user_id                              wav_file  \\\n",
       "0  test        -1    test  ./data/test/audio/clip_bd6d0fb25.wav   \n",
       "1  test        -1    test  ./data/test/audio/clip_3e7a56353.wav   \n",
       "2  test        -1    test  ./data/test/audio/clip_c5884a6cb.wav   \n",
       "3  test        -1    test  ./data/test/audio/clip_ebf0d1f7b.wav   \n",
       "4  test        -1    test  ./data/test/audio/clip_2f714f052.wav   \n",
       "\n",
       "                                                 raw  \n",
       "0  [[-57.2323073539, -62.8251292953, -75.84322783...  \n",
       "1  [[-58.4158758303, -52.5011356178, -53.86649120...  \n",
       "2  [[-73.8001558488, -76.330194253, -72.386112091...  \n",
       "3  [[-71.7484306025, -64.9348066752, -62.39292603...  \n",
       "4  [[-49.6434011746, -49.9060486303, -49.58891626...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T10:32:14.882322Z",
     "start_time": "2017-11-17T10:32:14.863617Z"
    },
    "_cell_guid": "c6d9b369-9979-4bcd-8540-4653e6544f84",
    "_uuid": "6a0bb3c22b7b5c43db0ec5673333ab3de8f08724"
   },
   "outputs": [],
   "source": [
    "batch_size = 64 \n",
    "\n",
    "def test_generator(test_batch_size,augment=False):\n",
    "    while True:\n",
    "        ids = list(range(test_df.shape[0]))\n",
    "        \n",
    "        for start in range(0, len(ids), test_batch_size):\n",
    "            x_batch = []\n",
    "            end = min(start + test_batch_size, len(ids))\n",
    "            i_test_batch = ids[start:end]\n",
    "#             this_paths = test_paths[start:end]\n",
    "#             for x in this_paths:\n",
    "            for i in i_test_batch:\n",
    "                if augment:\n",
    "                    x_batch.append(process_wav_file(test_df.loc[i],n_mels=256,augment=True).T) #,reshape=False,augment=augment,pval=0.5))\n",
    "                else:    \n",
    "                    x_batch.append(test_df.loc[i,'raw'].T)\n",
    "\n",
    "            x_batch = np.array(x_batch)\n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            \n",
    "            yield x_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-11-17T10:32:45.947Z"
    },
    "_cell_guid": "1fb8aed4-de12-43c5-84bf-b803e3d640fa",
    "_uuid": "631a38cb0013e5772f6987854145ad76ecf6c430"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2478/2478 [==============================] - 224s 90ms/step\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict_generator(test_generator(batch_size), int(np.ceil(len(test_paths)/float(batch_size))), verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"cache/predictions_{}.npy\".format(exp_name),predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = np.load(\"cache/predictions_conv2d_wlstm_pseudo_wval.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2478/2478 [==============================] - 1705s 688ms/step\n",
      "2478/2478 [==============================] - 1541s 622ms/step\n",
      "2478/2478 [==============================] - 1544s 623ms/step\n",
      "2476/2478 [============================>.] - ETA: 1s"
     ]
    }
   ],
   "source": [
    "tta_preds = []\n",
    "num_aug = 4\n",
    "for i in range(num_aug):\n",
    "    tta_preds.append(model.predict_generator(test_generator(batch_size,augment=True), int(np.ceil(len(test_paths)/float(batch_size))), verbose=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "\n",
    "# pickle.dump(tta_preds,open(\"cache/test_preds_aug_4x.pik\",'wb'),pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# tta_preds = pickle.load(open(\"cache/test_preds_aug_4x.pik\",'rb'))\n",
    "\n",
    "# tta_preds.append(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = np.mean(tta_preds,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158538, 12)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### blending"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predictions_pyramid_noaug = np.load('cache/predictions_pyramid_noaug.npy')\n",
    "# predictions_model_with_ae_base_drp2_1 = np.load('cache/predictions_model_with_ae_base_drp2_1.npy')\n",
    "# blend1 = np.load('cache/predictions_blend_dilated_conv1d_timek123_n_freqk48_pseudo_mixtimefreq1ds_plus_samewithlstm_plus_aebased_conv2d_finetuned.npy')\n",
    "# predictions_aebase_aug_drp3_finetune = np.load('cache/predictions_aebase_aug_drp3_finetune.npy')\n",
    "# predictions_conv_n_lstm = np.load(\"cache/predictions_dilated_conv_n_lstm.npy\")\n",
    "blend2 = np.load('cache/predictions_frqmaxpool256_plus_longblend1.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "blend3 = np.mean([predictions,\n",
    "                       blend2], axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"cache/predictions_{}.npy\".format(\"conv2dlstmpsuedo_plus_longblend1\"),blend3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2477/2478 [============================>.] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "# num_aug = 2 \n",
    "# for i in range(num_aug):\n",
    "#     predictions +=  model.predict_generator(test_generator(64,augment=True), int(np.ceil(len(test_paths)/64.)), verbose=1)\n",
    "    \n",
    "\n",
    "# predictions = predictions/(num_aug + 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### class assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T11:30:44.236246Z",
     "start_time": "2017-11-17T11:30:44.21858Z"
    },
    "_cell_guid": "b1cdab5c-9816-4690-87d8-de2c97cf0e7d",
    "_uuid": "24eb7e512eace4567494e0a8e356a826f4283c4d"
   },
   "outputs": [],
   "source": [
    "classes = np.argmax(predictions, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 12543.,   6661.,   5327.,   6491.,   5741.,   6487.,   7599.,\n",
       "          5891.,   5620.,  96178.]),\n",
       " array([  0. ,   1.1,   2.2,   3.3,   4.4,   5.5,   6.6,   7.7,   8.8,\n",
       "          9.9,  11. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAERlJREFUeJzt3GusXXWZx/Hvb1pRwAhFGoItmZLYaCqJAU6gDokxYKCA\nsbxQgpmRhhD7QlQ0Jlp800QlwYkRIVESApXiEJAgCY0WOw2XmHkBcgAjNwknXNvhcrRcvESx+syL\n/WdmT+nlz9mn7Pb0+0lO9lrP+q+1ngUn/Z1126kqJEnq8U/jbkCStP8wNCRJ3QwNSVI3Q0OS1M3Q\nkCR1MzQkSd0MDUlSN0NDktRtj6GRZF2Sl5I8PFQ7IsnmJE+0zwWtniRXJplK8pskJwyts6qNfyLJ\nqqH6iUkeautcmSS724ckaXyypzfCk3wU+CNwfVUd12r/DmyrqsuSrAEWVNXXk5wFfBE4CzgZuKKq\nTk5yBDAJTAAF3A+cWFUvJ/kV8CXgXmAjcGVV3b6rfezpgI488shasmTJDP5TSNKB6/777/9dVS3c\n07j5expQVb9MsmSH8krgY216PXA38PVWv74GSXRPksOTHN3Gbq6qbQBJNgMrktwNvKeq7mn164Fz\ngNt3s4/dWrJkCZOTk3saJkkakuSZnnEzvadxVFU936ZfAI5q04uA54bGbWm13dW37KS+u328SZLV\nSSaTTE5PT8/gcCRJPUa+Ed7OKvbqtx7uaR9VdXVVTVTVxMKFezy7kiTN0ExD48V22Yn2+VKrbwWO\nGRq3uNV2V1+8k/ru9iFJGpOZhsYG4I0noFYBtw3Vz29PUS0HXm2XmDYBpydZ0J6COh3Y1Ja9lmR5\ne2rq/B22tbN9SJLGZI83wpPcyOCG9JFJtgBrgcuAm5NcCDwDnNuGb2Tw5NQU8GfgAoCq2pbkW8B9\nbdw337gpDnweuA44mMEN8NtbfVf7kCSNyR4fud3fTExMlE9PSdJbk+T+qprY0zjfCJckdTM0JEnd\nDA1JUrc93giXJPVbsubnY9nv05ed/bbsxzMNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAk\ndTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAk\ndTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktRtpNBI\n8pUkjyR5OMmNSd6V5Ngk9yaZSvKTJAe1se9s81Nt+ZKh7VzS6o8nOWOovqLVppKsGaVXSdLoZhwa\nSRYBXwImquo4YB5wHvAd4PKqej/wMnBhW+VC4OVWv7yNI8mytt6HgBXAD5PMSzIP+AFwJrAM+Ewb\nK0kak1EvT80HDk4yHzgEeB44FbilLV8PnNOmV7Z52vLTkqTVb6qqv1bVU8AUcFL7maqqJ6vqdeCm\nNlaSNCYzDo2q2gp8F3iWQVi8CtwPvFJV29uwLcCiNr0IeK6tu72Nf+9wfYd1dlWXJI3JKJenFjD4\ny/9Y4H3AoQwuL73tkqxOMplkcnp6ehwtSNIBYZTLUx8Hnqqq6ar6G3ArcApweLtcBbAY2NqmtwLH\nALTlhwG/H67vsM6u6m9SVVdX1URVTSxcuHCEQ5Ik7c4oofEssDzJIe3exGnAo8BdwKfamFXAbW16\nQ5unLb+zqqrVz2tPVx0LLAV+BdwHLG1PYx3E4Gb5hhH6lSSNaP6eh+xcVd2b5BbgAWA78CBwNfBz\n4KYk3261a9sq1wI/TjIFbGMQAlTVI0luZhA424GLqurvAEm+AGxi8GTWuqp6ZKb9SpJGN+PQAKiq\ntcDaHcpPMnjyacexfwE+vYvtXApcupP6RmDjKD1KkmaPb4RLkroZGpKkboaGJKmboSFJ6mZoSJK6\nGRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6\nGRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6\nGRqSpG6GhiSpm6EhSepmaEiSuhkakqRuI4VGksOT3JLkt0keS/KRJEck2Zzkifa5oI1NkiuTTCX5\nTZIThrazqo1/IsmqofqJSR5q61yZJKP0K0kazahnGlcAv6iqDwIfBh4D1gB3VNVS4I42D3AmsLT9\nrAauAkhyBLAWOBk4CVj7RtC0MZ8bWm/FiP1KkkYw49BIchjwUeBagKp6vapeAVYC69uw9cA5bXol\ncH0N3AMcnuRo4Axgc1Vtq6qXgc3AirbsPVV1T1UVcP3QtiRJYzDKmcaxwDTwoyQPJrkmyaHAUVX1\nfBvzAnBUm14EPDe0/pZW2119y07qkqQxGSU05gMnAFdV1fHAn/i/S1EAtDOEGmEfXZKsTjKZZHJ6\nenpv706SDlijhMYWYEtV3dvmb2EQIi+2S0u0z5fa8q3AMUPrL2613dUX76T+JlV1dVVNVNXEwoUL\nRzgkSdLuzDg0quoF4LkkH2il04BHgQ3AG09ArQJua9MbgPPbU1TLgVfbZaxNwOlJFrQb4KcDm9qy\n15Isb09NnT+0LUnSGMwfcf0vAjckOQh4EriAQRDdnORC4Bng3DZ2I3AWMAX8uY2lqrYl+RZwXxv3\nzara1qY/D1wHHAzc3n4kSWMyUmhU1a+BiZ0sOm0nYwu4aBfbWQes20l9EjhulB4lSbPHN8IlSd0M\nDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0M\nDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0M\nDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVK3kUMjybwkDyb5WZs/Nsm9\nSaaS/CTJQa3+zjY/1ZYvGdrGJa3+eJIzhuorWm0qyZpRe5UkjWY2zjQuBh4bmv8OcHlVvR94Gbiw\n1S8EXm71y9s4kiwDzgM+BKwAftiCaB7wA+BMYBnwmTZWkjQmI4VGksXA2cA1bT7AqcAtbch64Jw2\nvbLN05af1savBG6qqr9W1VPAFHBS+5mqqier6nXgpjZWkjQmo55pfB/4GvCPNv9e4JWq2t7mtwCL\n2vQi4DmAtvzVNv5/6zuss6u6JGlMZhwaST4BvFRV989iPzPtZXWSySST09PT425HkuasUc40TgE+\nmeRpBpeOTgWuAA5PMr+NWQxsbdNbgWMA2vLDgN8P13dYZ1f1N6mqq6tqoqomFi5cOMIhSZJ2Z8ah\nUVWXVNXiqlrC4Eb2nVX1r8BdwKfasFXAbW16Q5unLb+zqqrVz2tPVx0LLAV+BdwHLG1PYx3U9rFh\npv1KkkY3f89D3rKvAzcl+TbwIHBtq18L/DjJFLCNQQhQVY8kuRl4FNgOXFRVfwdI8gVgEzAPWFdV\nj+yFfiVJnWYlNKrqbuDuNv0kgyefdhzzF+DTu1j/UuDSndQ3Ahtno0dJ0uh8I1yS1M3QkCR1MzQk\nSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQk\nSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQk\nSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHWbcWgkOSbJXUkeTfJIkotb/Ygkm5M8\n0T4XtHqSXJlkKslvkpwwtK1VbfwTSVYN1U9M8lBb58okGeVgJUmjGeVMYzvw1apaBiwHLkqyDFgD\n3FFVS4E72jzAmcDS9rMauAoGIQOsBU4GTgLWvhE0bcznhtZbMUK/kqQRzTg0qur5qnqgTf8BeAxY\nBKwE1rdh64Fz2vRK4PoauAc4PMnRwBnA5qraVlUvA5uBFW3Ze6rqnqoq4PqhbUmSxmBW7mkkWQIc\nD9wLHFVVz7dFLwBHtelFwHNDq21ptd3Vt+ykLkkak5FDI8m7gZ8CX66q14aXtTOEGnUfHT2sTjKZ\nZHJ6enpv706SDlgjhUaSdzAIjBuq6tZWfrFdWqJ9vtTqW4FjhlZf3Gq7qy/eSf1NqurqqpqoqomF\nCxeOckiSpN0Y5empANcCj1XV94YWbQDeeAJqFXDbUP389hTVcuDVdhlrE3B6kgXtBvjpwKa27LUk\ny9u+zh/aliRpDOaPsO4pwGeBh5L8utW+AVwG3JzkQuAZ4Ny2bCNwFjAF/Bm4AKCqtiX5FnBfG/fN\nqtrWpj8PXAccDNzefiRJYzLj0Kiq/wJ29d7EaTsZX8BFu9jWOmDdTuqTwHEz7VGSNLt8I1yS1M3Q\nkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3Ub5wsI5Z8man49l\nv09fdvZY9itJb5VnGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuvhEu\nHQD8tgPNFs80JEndPNPYB4zrr0DwL0FJb42hobE4EINynMcszRZDQwcc//F++xyIfxzMdYbGAc5/\nQDVX+bu9d3gjXJLUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd32+dBIsiLJ40mmkqwZdz+S\ndCDbp0MjyTzgB8CZwDLgM0mWjbcrSTpw7dOhAZwETFXVk1X1OnATsHLMPUnSAWtfD41FwHND81ta\nTZI0BnPiu6eSrAZWt9k/Jnl8hps6Evjd7HS1z5nLxwZz+/g8tv3X23Z8+c7Im/jnnkH7emhsBY4Z\nml/cav9PVV0NXD3qzpJMVtXEqNvZF83lY4O5fXwe2/5rLh7fvn556j5gaZJjkxwEnAdsGHNPknTA\n2qfPNKpqe5IvAJuAecC6qnpkzG1J0gFrnw4NgKraCGx8m3Y38iWufdhcPjaY28fnse2/5tzxparG\n3YMkaT+xr9/TkCTtQwyNZq5+XUmSY5LcleTRJI8kuXjcPc22JPOSPJjkZ+PuZbYlOTzJLUl+m+Sx\nJB8Zd0+zJclX2u/kw0luTPKucfc0iiTrkryU5OGh2hFJNid5on0uGGePs8HQYM5/Xcl24KtVtQxY\nDlw0h47tDRcDj427ib3kCuAXVfVB4MPMkeNMsgj4EjBRVccxeNDlvPF2NbLrgBU71NYAd1TVUuCO\nNr9fMzQG5uzXlVTV81X1QJv+A4N/dObMW/VJFgNnA9eMu5fZluQw4KPAtQBV9XpVvTLermbVfODg\nJPOBQ4D/HnM/I6mqXwLbdiivBNa36fXAOW9rU3uBoTFwQHxdSZIlwPHAvePtZFZ9H/ga8I9xN7IX\nHAtMAz9ql9+uSXLouJuaDVW1Ffgu8CzwPPBqVf3neLvaK46qqufb9AvAUeNsZjYYGgeIJO8Gfgp8\nuapeG3c/syHJJ4CXqur+cfeyl8wHTgCuqqrjgT8xBy5vALRr+ysZBOP7gEOT/Nt4u9q7avCo6n7/\nuKqhMdD1dSX7qyTvYBAYN1TVrePuZxadAnwyydMMLimemuQ/xtvSrNoCbKmqN84Mb2EQInPBx4Gn\nqmq6qv4G3Ar8y5h72hteTHI0QPt8acz9jMzQGJizX1eSJAyuiT9WVd8bdz+zqaouqarFVbWEwf+z\nO6tqzvy1WlUvAM8l+UArnQY8OsaWZtOzwPIkh7Tf0dOYIzf5d7ABWNWmVwG3jbGXWbHPvxH+dpjj\nX1dyCvBZ4KEkv261b7Q37bXv+yJwQ/tj5knggjH3Myuq6t4ktwAPMHjC70H287enk9wIfAw4MskW\nYC1wGXBzkguBZ4Bzx9fh7PCNcElSNy9PSZK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuh\nIUnq9j+U59VghvdkyQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6603f6e8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(classes,bins=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 12300.,   6025.,   5316.,   6262.,   5586.,   6258.,   6430.,\n",
       "          5676.,   5577.,  99108.]),\n",
       " array([  0. ,   1.1,   2.2,   3.3,   4.4,   5.5,   6.6,   7.7,   8.8,\n",
       "          9.9,  11. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEQdJREFUeJzt3W2sXWWZxvH/Na0oYuRFGqItmZJMo6kkBmywDokx1kBR\nY/mgBDMjDSH2g+BbTJzilyYqE0yML0yUhEi1OAQkSEKjVaZBjJkPIAcxIiDhBETaKXC0vDgaxeo9\nH/bDzJ5y2j6efco6Pf3/kp291r2etda9oKfXWS97N1WFJEk9/m7oBiRJRw5DQ5LUzdCQJHUzNCRJ\n3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSt6VDNzDfTj755Fq5cuXQbUjSEeWee+75TVUtO9S4RRca\nK1euZGpqaug2JOmIkuSxnnFenpIkdTM0JEndDA1JUrdDhkaSrUmeSvKLsdpJSXYmebi9n9jqSXJV\nkukkP09y5tg6G9v4h5NsHKu/Ocl9bZ2rkuRg+5AkDafnTOObwPr9apuB26tqFXB7mwc4D1jVXpuA\nq2EUAMAW4C3AWcCWsRC4GvjQ2HrrD7EPSdJADhkaVfVjYO9+5Q3Atja9DTh/rH5djdwJnJDktcC5\nwM6q2ltVTwM7gfVt2aur6s4a/WtQ1+23rdn2IUkayFzvaZxSVXva9BPAKW16OfD42LhdrXaw+q5Z\n6gfbhyRpIBPfCG9nCIf134w91D6SbEoylWRqZmbmcLYiSUe1uYbGk+3SEu39qVbfDZw6Nm5Fqx2s\nvmKW+sH28SJVdU1VramqNcuWHfIDjZKkOZrrJ8K3AxuBK9v7rWP1y5LcyOim97NVtSfJbcC/jt38\nPge4vKr2JnkuyVrgLuAi4N8OsQ9JWrBWbv7eIPv91ZXvfkn2c8jQSHID8Hbg5CS7GD0FdSVwU5JL\ngMeAC9rwHcC7gGngD8DFAC0cPgvc3cZ9pqpeuLn+YUZPaB0LfL+9OMg+JEkDOWRoVNUHDrBo3Sxj\nC7j0ANvZCmydpT4FnD5L/bez7UOSNBw/ES5J6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRu\nhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRu\nhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRu\nhoYkqZuhIUnqNlFoJPlEkvuT/CLJDUlekeS0JHclmU7y7STHtLEvb/PTbfnKse1c3uoPJTl3rL6+\n1aaTbJ6kV0nS5OYcGkmWAx8F1lTV6cAS4ELg88CXquofgKeBS9oqlwBPt/qX2jiSrG7rvRFYD3wt\nyZIkS4CvAucBq4EPtLGSpIFMenlqKXBskqXAK4E9wDuAm9vybcD5bXpDm6ctX5ckrX5jVf2pqh4F\npoGz2mu6qh6pqueBG9tYSdJA5hwaVbUb+ALwa0Zh8SxwD/BMVe1rw3YBy9v0cuDxtu6+Nv414/X9\n1jlQXZI0kEkuT53I6Df/04DXAccxurz0kkuyKclUkqmZmZkhWpCko8Ikl6feCTxaVTNV9WfgFuBs\n4IR2uQpgBbC7Te8GTgVoy48Hfjte32+dA9VfpKquqao1VbVm2bJlExySJOlgJgmNXwNrk7yy3ZtY\nBzwA3AG8r43ZCNzapre3edryH1ZVtfqF7emq04BVwE+Au4FV7WmsYxjdLN8+Qb+SpAktPfSQ2VXV\nXUluBn4K7APuBa4BvgfcmORzrXZtW+Va4FtJpoG9jEKAqro/yU2MAmcfcGlV/QUgyWXAbYyezNpa\nVffPtV9J0uTmHBoAVbUF2LJf+RFGTz7tP/aPwPsPsJ0rgCtmqe8AdkzSoyRp/viJcElSN0NDktTN\n0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN\n0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN\n0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVK3iUIjyQlJbk7yyyQPJnlrkpOS7EzycHs/\nsY1NkquSTCf5eZIzx7azsY1/OMnGsfqbk9zX1rkqSSbpV5I0mUnPNL4C/KCq3gC8CXgQ2AzcXlWr\ngNvbPMB5wKr22gRcDZDkJGAL8BbgLGDLC0HTxnxobL31E/YrSZrAnEMjyfHA24BrAarq+ap6BtgA\nbGvDtgHnt+kNwHU1cidwQpLXAucCO6tqb1U9DewE1rdlr66qO6uqgOvGtiVJGsAkZxqnATPAN5Lc\nm+TrSY4DTqmqPW3ME8ApbXo58PjY+rta7WD1XbPUXyTJpiRTSaZmZmYmOCRJ0sFMEhpLgTOBq6vq\nDOD3/N+lKADaGUJNsI8uVXVNVa2pqjXLli073LuTpKPWJKGxC9hVVXe1+ZsZhciT7dIS7f2ptnw3\ncOrY+ita7WD1FbPUJUkDmXNoVNUTwONJXt9K64AHgO3AC09AbQRubdPbgYvaU1RrgWfbZazbgHOS\nnNhugJ8D3NaWPZdkbXtq6qKxbUmSBrB0wvU/Alyf5BjgEeBiRkF0U5JLgMeAC9rYHcC7gGngD20s\nVbU3yWeBu9u4z1TV3jb9YeCbwLHA99tLkjSQiUKjqn4GrJll0bpZxhZw6QG2sxXYOkt9Cjh9kh4l\nSfPHT4RLkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuh\nIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuh\nIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuk0cGkmWJLk3yXfb\n/GlJ7koyneTbSY5p9Ze3+em2fOXYNi5v9YeSnDtWX99q00k2T9qrJGky83Gm8THgwbH5zwNfqqp/\nAJ4GLmn1S4CnW/1LbRxJVgMXAm8E1gNfa0G0BPgqcB6wGvhAGytJGshEoZFkBfBu4OttPsA7gJvb\nkG3A+W16Q5unLV/Xxm8AbqyqP1XVo8A0cFZ7TVfVI1X1PHBjGytJGsikZxpfBj4F/LXNvwZ4pqr2\ntfldwPI2vRx4HKAtf7aN/9/6fuscqP4iSTYlmUoyNTMzM+EhSZIOZM6hkeQ9wFNVdc889jMnVXVN\nVa2pqjXLli0buh1JWrSWTrDu2cB7k7wLeAXwauArwAlJlraziRXA7jZ+N3AqsCvJUuB44Ldj9ReM\nr3OguiRpAHM+06iqy6tqRVWtZHQj+4dV9U/AHcD72rCNwK1tenubpy3/YVVVq1/Ynq46DVgF/AS4\nG1jVnsY6pu1j+1z7lSRNbpIzjQP5F+DGJJ8D7gWubfVrgW8lmQb2MgoBqur+JDcBDwD7gEur6i8A\nSS4DbgOWAFur6v7D0K8kqdO8hEZV/Qj4UZt+hNGTT/uP+SPw/gOsfwVwxSz1HcCO+ehRkjQ5PxEu\nSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6Eh\nSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6Eh\nSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKnbnEMjyalJ7kjyQJL7k3ys1U9KsjPJ\nw+39xFZPkquSTCf5eZIzx7a1sY1/OMnGsfqbk9zX1rkqSSY5WEnSZCY509gHfLKqVgNrgUuTrAY2\nA7dX1Srg9jYPcB6wqr02AVfDKGSALcBbgLOALS8ETRvzobH11k/QryRpQnMOjaraU1U/bdO/Ax4E\nlgMbgG1t2Dbg/Da9AbiuRu4ETkjyWuBcYGdV7a2qp4GdwPq27NVVdWdVFXDd2LYkSQOYl3saSVYC\nZwB3AadU1Z626AnglDa9HHh8bLVdrXaw+q5Z6pKkgUwcGkleBXwH+HhVPTe+rJ0h1KT76OhhU5Kp\nJFMzMzOHe3eSdNSaKDSSvIxRYFxfVbe08pPt0hLt/alW3w2cOrb6ilY7WH3FLPUXqaprqmpNVa1Z\ntmzZJIckSTqISZ6eCnAt8GBVfXFs0XbghSegNgK3jtUvak9RrQWebZexbgPOSXJiuwF+DnBbW/Zc\nkrVtXxeNbUuSNIClE6x7NvBB4L4kP2u1TwNXAjcluQR4DLigLdsBvAuYBv4AXAxQVXuTfBa4u437\nTFXtbdMfBr4JHAt8v70kSQOZc2hU1X8CB/rcxLpZxhdw6QG2tRXYOkt9Cjh9rj1KkuaXnwiXJHUz\nNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUrdJvrBw0Vm5+XuD\n7PdXV757kP1K0t/KMw1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0ND\nktTN0JAkdfO7p6SjgN+rpvliaCwAQ/1Aw3A/1B6zdGQyNHTU8S/vl47/rRcfQ+Mo5w+1pL+FN8Il\nSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUrcFHxpJ1id5KMl0ks1D9yNJR7MFHRpJlgBfBc4DVgMf\nSLJ62K4k6ei1oEMDOAuYrqpHqup54EZgw8A9SdJRa6GHxnLg8bH5Xa0mSRrAovgakSSbgE1t9r+T\nPDTHTZ0M/GZ+ulpwFvOxweI+Po/tyPWSHV8+P/Em/r5n0EIPjd3AqWPzK1rt/6mqa4BrJt1Zkqmq\nWjPpdhaixXxssLiPz2M7ci3G41vol6fuBlYlOS3JMcCFwPaBe5Kko9aCPtOoqn1JLgNuA5YAW6vq\n/oHbkqSj1oIODYCq2gHseIl2N/ElrgVsMR8bLO7j89iOXIvu+FJVQ/cgSTpCLPR7GpKkBcTQaBbr\n15UkOTXJHUkeSHJ/ko8N3dN8S7Ikyb1Jvjt0L/MtyQlJbk7yyyQPJnnr0D3NlySfaH8mf5HkhiSv\nGLqnSSTZmuSpJL8Yq52UZGeSh9v7iUP2OB8MDRb915XsAz5ZVauBtcCli+jYXvAx4MGhmzhMvgL8\noKreALyJRXKcSZYDHwXWVNXpjB50uXDYrib2TWD9frXNwO1VtQq4vc0f0QyNkUX7dSVVtaeqftqm\nf8foL51F86n6JCuAdwNfH7qX+ZbkeOBtwLUAVfV8VT0zbFfzailwbJKlwCuB/xq4n4lU1Y+BvfuV\nNwDb2vQ24PyXtKnDwNAYOSq+riTJSuAM4K5hO5lXXwY+Bfx16EYOg9OAGeAb7fLb15McN3RT86Gq\ndgNfAH4N7AGerar/GLarw+KUqtrTpp8AThmymflgaBwlkrwK+A7w8ap6buh+5kOS9wBPVdU9Q/dy\nmCwFzgSurqozgN+zCC5vALRr+xsYBePrgOOS/POwXR1eNXpU9Yh/XNXQGOn6upIjVZKXMQqM66vq\nlqH7mUdnA+9N8itGlxTfkeTfh21pXu0CdlXVC2eGNzMKkcXgncCjVTVTVX8GbgH+ceCeDocnk7wW\noL0/NXA/EzM0Rhbt15UkCaNr4g9W1ReH7mc+VdXlVbWiqlYy+n/2w6paNL+tVtUTwONJXt9K64AH\nBmxpPv0aWJvkle3P6DoWyU3+/WwHNrbpjcCtA/YyLxb8J8JfCov860rOBj4I3JfkZ6326fZJey18\nHwGub7/MPAJcPHA/86Kq7kpyM/BTRk/43csR/unpJDcAbwdOTrIL2AJcCdyU5BLgMeCC4TqcH34i\nXJLUzctTkqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6/Q/WOs2sgMLJEgAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4ed25995d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(classes,bins=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T11:31:11.212517Z",
     "start_time": "2017-11-17T11:31:10.786357Z"
    },
    "_cell_guid": "1da523cf-fdbf-4ab1-9300-0147155aa247",
    "_uuid": "f25d4e626202aa115bd0460f4de8d07f9727c83e"
   },
   "outputs": [],
   "source": [
    "### last batch will contain padding, so remove duplicates\n",
    "submission = dict()\n",
    "for i in range(len(test_paths)):\n",
    "    fname, label = os.path.basename(test_paths[i]), id2name[classes[i]]\n",
    "    submission[fname] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T11:32:05.154527Z",
     "start_time": "2017-11-17T11:32:04.983371Z"
    },
    "_cell_guid": "9a95d147-3f4b-4386-8597-5fa60be43542",
    "_uuid": "bdf63bce43a0525a02ac18ca3f90aeba06ce6e99"
   },
   "outputs": [],
   "source": [
    "with open('subm/{}_tta.csv'.format(exp_name), 'w') as fout: #_blend_conv1dlstm_and_aebased_conv2d_finetuned\n",
    "    fout.write('fname,label\\n')\n",
    "    for fname, label in submission.items():\n",
    "        fout.write('{},{}\\n'.format(fname, label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "_cell_guid": "8bea6850-15c6-44e7-bdb4-9555ad196f85",
    "_uuid": "555315ef622793711ff5643928dac874c8cb0ed2"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='subm/conv2d_wlstm_pseudo_wval_plus_blend2.csv' target='_blank'>subm/conv2d_wlstm_pseudo_wval_plus_blend2.csv</a><br>"
      ],
      "text/plain": [
       "/home/ubuntu/keyword_spotting/subm/conv2d_wlstm_pseudo_wval_plus_blend2.csv"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.lib.display import FileLink\n",
    "\n",
    "FileLink('subm/{}_plus_blend2.csv'.format(exp_name))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}