triplet_loss_starter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:29.196238Z",
     "start_time": "2017-11-17T09:03:28.644004Z"
    },
    "_cell_guid": "679e0d3e-646d-4e96-9eb0-b362d8c6e51f",
    "_uuid": "0d05e5ce89af3e25d1c1fb244d021a1cfa1a058c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import array \n",
    "\n",
    "from pydub import AudioSegment\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Input, Conv2D, Flatten, GlobalMaxPooling1D, MaxPooling2D, Activation, BatchNormalization, GlobalAveragePooling2D, GlobalMaxPool2D, concatenate, Dense, Dropout\n",
    "from keras.optimizers import Adam\n",
    "from tensorflow.python.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Input, GRU, RepeatVector, BatchNormalization, TimeDistributed, Conv1D\n",
    "from keras.layers import GlobalAveragePooling1D, LSTM, MaxPooling1D, CuDNNLSTM, Bidirectional\n",
    "from keras import backend as K\n",
    "from keras.layers import  Conv2D, MaxPooling2D, UpSampling2D, Lambda, Reshape\n",
    "import keras\n",
    "from keras.layers import AveragePooling1D\n",
    "\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "import matplotlib.pyplot as plt\n",
    "import numba\n",
    "from keras import losses\n",
    "import dask.dataframe as dd\n",
    "from dask.multiprocessing import get\n",
    "from keras.layers import Embedding, Flatten, Input, merge\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "POSSIBLE_LABELS = 'yes no up down left right on off stop go silence unknown'.split()\n",
    "id2name = {i: name for i, name in enumerate(POSSIBLE_LABELS)}\n",
    "name2id = {name: i for i, name in id2name.items()}\n",
    "len(id2name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### create validation dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pickle.load( open(\"cache/train_df.pik\",\"rb\"))\n",
    "valid_df = pickle.load( open(\"cache/valid_df.pik\",\"rb\"))\n",
    "silent_df = pickle.load(open(\"cache/silent_df.pik\",\"rb\"))\n",
    "\n",
    "unknown_df = pickle.load(open(\"cache/unknown_df.pik\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train_df[\"label_user\"] = train_df[\"label\"]+train_df[\"user_id\"]\n",
    "# valid_df[\"label_user\"] = valid_df[\"label\"]+valid_df[\"user_id\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "silent_df[\"label_id\"] = name2id[\"silence\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "        this_train = train_df.groupby('label_id').apply(lambda x: x.sample(n = 2000))\n",
    "        extra_data_size = int(this_train.shape[0]* 0.1)\n",
    "        this_train = pd.concat([silent_df.sample(extra_data_size),\n",
    "                                this_train,\n",
    "                                unknown_df.sample(extra_data_size*2)],axis=0 )\n",
    "        \n",
    "        this_train.reset_index(drop=True,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11    4000\n",
       "10    2000\n",
       "9     2000\n",
       "8     2000\n",
       "7     2000\n",
       "6     2000\n",
       "5     2000\n",
       "4     2000\n",
       "3     2000\n",
       "2     2000\n",
       "1     2000\n",
       "0     2000\n",
       "Name: label_id, dtype: int64"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "this_train.label_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_generator(train_batch_size):\n",
    "    \n",
    "    \n",
    "\n",
    "    while True:\n",
    "        \n",
    "        this_train = train_df.groupby('label_id').apply(lambda x: x.sample(n = 2000))\n",
    "        extra_data_size = int(this_train.shape[0]* 0.1)\n",
    "        this_train = pd.concat([silent_df.sample(extra_data_size),\n",
    "                                this_train,\n",
    "                                unknown_df.sample(extra_data_size*2)],axis=0 )\n",
    "        \n",
    "        this_train.reset_index(drop=True,inplace=True)\n",
    "        \n",
    "        \n",
    "        x_batch_A_1 = []\n",
    "        x_batch_A_2 = []\n",
    "        x_batch_B = []\n",
    "        y_batch = []\n",
    "            \n",
    "        for i in range(train_batch_size):\n",
    "\n",
    "\n",
    "            #choose label A, and something else (B)\n",
    "            first_label = np.random.randint(0,len(id2name))\n",
    "            second_label = np.random.choice([i for i in list(id2name.keys()) if i !=first_label ]) #todo:hard negative\n",
    "\n",
    "            \n",
    "            \n",
    "            #choose a random label A\n",
    "            similars = this_train[this_train.label_id == first_label]\n",
    "            #choose 2 distinct utterances from label A \n",
    "            sim_1, sim_2 = np.random.choice(range(len(similars)),2,replace=False)\n",
    "            A_1 , A_2 = similars.raw.iloc[sim_1], similars.raw.iloc[sim_2] \n",
    "\n",
    "            # choose a different utterance from second label B\n",
    "            diffs = this_train[this_train.label_id == second_label]\n",
    "            B = diffs.raw.iloc[np.random.choice(range(len(diffs)))]\n",
    "\n",
    "\n",
    "            # create triplet\n",
    "            x_batch_A_1.append(A_1.T)\n",
    "            x_batch_A_2.append(A_2.T)\n",
    "            x_batch_B.append(B.T)\n",
    "#                 x_batch.append([A_1.T,A_2.T,B.T])\n",
    "            y_batch.append(similars.label_id.values[0])\n",
    "\n",
    "        x_batch_A_1 = 1.- np.array(x_batch_A_1)/-80.\n",
    "        x_batch_A_2 = 1.- np.array(x_batch_A_2)/-80.\n",
    "        x_batch_B = 1.- np.array(x_batch_B)/-80.\n",
    "\n",
    "#         y_batch = np.ones((train_batch_size)) \n",
    "        y_batch = to_categorical(y_batch, num_classes = len(POSSIBLE_LABELS))\n",
    "    \n",
    "        yield [x_batch_A_1,x_batch_A_2,x_batch_B], [np.ones((train_batch_size)) ,y_batch]# [y_batch,y_batch,y_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 216 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.36405783023470067, 0.37263880476216477, 0.29780967434027689)"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = train_generator(64)\n",
    "%time tt = next(t)[0]\n",
    "tt[0].mean(), tt[1].mean(), tt[2].mean(), "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64, 32, 128)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "tt[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_generator(train_batch_size):\n",
    "\n",
    "    while True:\n",
    "        \n",
    "        this_train = valid_df\n",
    "        this_train.reset_index(drop=True,inplace=True)\n",
    "        \n",
    "        \n",
    "        x_batch_A_1 = []\n",
    "        x_batch_A_2 = []\n",
    "        x_batch_B = []\n",
    "        y_batch = []\n",
    "\n",
    "    \n",
    "            \n",
    "        for i in range(train_batch_size):\n",
    "\n",
    "\n",
    "            #choose label A, and something else (B)\n",
    "            first_label = np.random.randint(0,len(id2name))\n",
    "            second_label = np.random.choice([i for i in list(id2name.keys()) if i !=first_label ]) #todo:hard negative\n",
    "\n",
    "            \n",
    "            \n",
    "            #choose a random label A\n",
    "            similars = this_train[this_train.label_id == first_label]\n",
    "            #choose 2 distinct utterances from label A \n",
    "            sim_1, sim_2 = np.random.choice(range(len(similars)),2,replace=False)\n",
    "            A_1 , A_2 = similars.raw.iloc[sim_1], similars.raw.iloc[sim_2] \n",
    "\n",
    "            # choose a different utterance from second label B\n",
    "            diffs = this_train[this_train.label_id == second_label]\n",
    "            B = diffs.raw.iloc[np.random.choice(range(len(diffs)))]\n",
    "\n",
    "\n",
    "            # create triplet\n",
    "            x_batch_A_1.append(A_1.T)\n",
    "            x_batch_A_2.append(A_2.T)\n",
    "            x_batch_B.append(B.T)\n",
    "#                 x_batch.append([A_1.T,A_2.T,B.T])\n",
    "            y_batch.append(similars.label_id.values[0])\n",
    "\n",
    "        x_batch_A_1 = 1.- np.array(x_batch_A_1)/-80.\n",
    "        x_batch_A_2 = 1.- np.array(x_batch_A_2)/-80.\n",
    "        x_batch_B = 1.- np.array(x_batch_B)/-80.\n",
    "\n",
    "#         y_batch = np.ones((train_batch_size)) \n",
    "        y_batch = to_categorical(y_batch, num_classes = len(POSSIBLE_LABELS))\n",
    "    \n",
    "        yield [x_batch_A_1,x_batch_A_2,x_batch_B], [np.ones((train_batch_size)) ,y_batch]# [y_batch,y_batch,y_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = valid_generator(64)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 132 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(64, 32, 128)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time next(v)[0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_1,X_2,X_3, y_triplet,y_target = [], [],[], [],[]\n",
    "\n",
    "for i in range( 150):\n",
    "    [x1,x2,x3], [y1,y2] = next(v)\n",
    "    X_1.append(x1) \n",
    "    X_2.append(x2) \n",
    "    X_3.append(x3) \n",
    "    \n",
    "    y_triplet.append(y1)\n",
    "    y_target.append(y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9600,)"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.hstack(y_triplet).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_1 = np.vstack(X_1)\n",
    "X_2 = np.vstack(X_2)\n",
    "X_3 = np.vstack(X_3)\n",
    "y_target = np.vstack(y_target)\n",
    "y_triplet = np.hstack(y_triplet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((9600, 32, 128), (9600,))"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_1.shape, y_triplet.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_relu(x):\n",
    "    x = BatchNormalization()(x)    \n",
    "    x = Activation('relu')(x)\n",
    "    \n",
    "    return x "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "timesteps, input_dim , latent_dim = 32,128, 128\n",
    "p = .0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# def triplet_loss(y_true, y_pred):\n",
    "#     mse1 = losses.mean_squared_error(y_pred[0],y_pred[1]) #K.sqrt(y_pred[0] - y_pred[1]).mean(axis=-1)\n",
    "#     mse2 = losses.mean_squared_error(y_pred[0],y_pred[2])#K.sqr(y_pred[0] - y_pred[2]).mean(axis=-1)\n",
    "    \n",
    "#     return K.maximum(0., mse1 - mse2 + 1) - y_true[0]*0\n",
    "\n",
    "\n",
    "# def triplet_loss( y_true, y_pred):\n",
    "\n",
    "#     embeddings = K.reshape(y_pred, (-1, 3, latent_dim))\n",
    "\n",
    "#     positive_distance = K.mean(K.square(embeddings[:,0] - embeddings[:,1]),axis=-1)\n",
    "#     negative_distance = K.mean(K.square(embeddings[:,0] - embeddings[:,2]),axis=-1)\n",
    "#     return K.mean(K.maximum(0.0, positive_distance - negative_distance + 1.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "def identity_loss(y_true, y_pred):\n",
    "\n",
    "    return K.mean(y_pred - 0 * y_true)\n",
    "\n",
    "\n",
    "\n",
    "def bpr_triplet_loss(X):\n",
    "\n",
    "    positive_item_latent, negative_item_latent, user_latent = X\n",
    "\n",
    "    # BPR loss\n",
    "    loss = 1.0 - K.sigmoid(\n",
    "        K.sum(user_latent * positive_item_latent, axis=-1, keepdims=True) -\n",
    "        K.sum(user_latent * negative_item_latent, axis=-1, keepdims=True))\n",
    "\n",
    "    return loss\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def embedder(filter_size=2):\n",
    "    v_input = Input(shape=( input_dim,timesteps))\n",
    "    x = BatchNormalization()(v_input)\n",
    "    \n",
    "\n",
    "    \n",
    "    x = Conv1D(64,filter_size,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv1D(64,filter_size,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    \n",
    "    \n",
    "    x  = MaxPooling1D(2)(x) \n",
    "    \n",
    "    x = Conv1D(128,filter_size,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv1D(128,filter_size,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    x  = AveragePooling1D(2)(x)   \n",
    "    \n",
    "#     x = Bidirectional(CuDNNLSTM(128,return_sequences=False))(x)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     x = Dense(128,activation='relu')(x)\n",
    "    \n",
    "    x = GlobalMaxPooling1D()(x)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "#     x = BatchNormalization()(v_input)\n",
    "    \n",
    "#     x = Conv1D(64,filter_size,padding='same')(x)\n",
    "#     x = batch_relu(x)\n",
    "#     x = Conv1D(64,filter_size,padding='same')(x)\n",
    "#     x = batch_relu(x)\n",
    "    \n",
    "#     x = Conv1D(128,3,padding='same')(x)\n",
    "#     x = batch_relu(x)\n",
    "#     x = Conv1D(128,3,padding='same')(x)\n",
    "#     x = batch_relu(x)\n",
    "       \n",
    "#     x = MaxPooling1D()(x)\n",
    "    \n",
    "#     x = Conv1D(256,3,padding='same')(x)\n",
    "#     x = batch_relu(x)\n",
    "#     x = Bidirectional(CuDNNLSTM(128,return_sequences=False))(x)\n",
    "\n",
    "    return Model(v_input, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Armin\\Anaconda2\\envs\\cx\\lib\\site-packages\\ipykernel_launcher.py:23: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "C:\\Users\\Armin\\Anaconda2\\envs\\cx\\lib\\site-packages\\keras\\legacy\\layers.py:458: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  name=name)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "xA1 = Input(shape=(timesteps, input_dim)) #1 channel, 99 time, 161 freqs # S : np.ndarray [shape=(n_mels, t)]\n",
    "xA2 = Input(shape=(timesteps, input_dim)) #1 channel, 99 time, 161 freqs # S : np.ndarray [shape=(n_mels, t)]\n",
    "xB = Input(shape=(timesteps, input_dim)) #1 channel, 99 time, 161 freqs # S : np.ndarray [shape=(n_mels, t)]\n",
    "\n",
    "xA1_freq = Reshape((input_dim, timesteps))(xA1)\n",
    "xA2_freq = Reshape((input_dim, timesteps))(xA2)\n",
    "xB_freq = Reshape((input_dim, timesteps))(xB)\n",
    "\n",
    "\n",
    "\n",
    "embedding = embedder(5)\n",
    "\n",
    "xA1_freq = embedding(xA1_freq)\n",
    "xA2_freq = embedding(xA2_freq)\n",
    "xB_freq = embedding(xB_freq)\n",
    "\n",
    "triplet_loss_output = merge(\n",
    "    [xA1_freq, xB_freq, xA2_freq],\n",
    "    mode=bpr_triplet_loss,\n",
    "    name='triplet_loss_output',\n",
    "    output_shape=(1,))\n",
    "\n",
    "\n",
    "x = add([xA1_freq,xA2_freq])\n",
    "x = Dense(128, activation = 'relu')(x) \n",
    "x = Dropout(0.3)(x)\n",
    "x = Dense(len(POSSIBLE_LABELS), activation = 'softmax', name='targets')(x)\n",
    "\n",
    "\n",
    "model = Model(\n",
    "        inputs=[xA1, xA2, xB],\n",
    "        outputs=[triplet_loss_output,x]) #concatenate([xA1_freq,xA2_freq,xB_freq],axis=0)\n",
    "        \n",
    "model.compile(Adam(lr=1e-3),\n",
    "              loss={'targets': 'categorical_crossentropy', 'triplet_loss_output': identity_loss},\n",
    "              loss_weights={'targets': 1., 'triplet_loss_output': 0.5})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_17 (InputLayer)           (None, 32, 128)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_19 (InputLayer)           (None, 32, 128)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_18 (InputLayer)           (None, 32, 128)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "reshape_13 (Reshape)            (None, 128, 32)      0           input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_15 (Reshape)            (None, 128, 32)      0           input_19[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_14 (Reshape)            (None, 128, 32)      0           input_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "model_8 (Model)                 (None, 128)          155648      reshape_13[0][0]                 \n",
      "                                                                 reshape_14[0][0]                 \n",
      "                                                                 reshape_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 128)          0           model_8[1][0]                    \n",
      "                                                                 model_8[2][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 128)          16512       add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 128)          0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "triplet_loss_output (Merge)     (None, 1)            0           model_8[1][0]                    \n",
      "                                                                 model_8[3][0]                    \n",
      "                                                                 model_8[2][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "targets (Dense)                 (None, 12)           1548        dropout_4[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 173,708\n",
      "Trainable params: 172,876\n",
      "Non-trainable params: 832\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_name = \"triplet_loss_starter\" \n",
    "callbacks = [EarlyStopping(monitor='val_loss',\n",
    "                           patience=5,\n",
    "                           verbose=1),\n",
    "             \n",
    "             ReduceLROnPlateau(monitor='val_loss',\n",
    "                               factor=0.1,\n",
    "                               patience=3,\n",
    "                               verbose=1,\n",
    "                               epsilon=0.01,\n",
    "                              min_lr=1e-5),\n",
    "             ModelCheckpoint(monitor='val_loss',\n",
    "                             filepath='weights/{}.hdf5'.format(exp_name),\n",
    "                             save_best_only=True,\n",
    "                             save_weights_only=True,\n",
    "                             mode='min')            ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "freqconv\n",
    "\n",
    "Epoch 15/100\n",
    "905/905 [==============================] - 94s 104ms/step - loss: 0.0247 - acc: 0.0245 - val_loss: 0.0230 - val_acc: 0.0236\n",
    "\n",
    "\n",
    "timeconv wlstm\n",
    "Epoch 17/100\n",
    "905/905 [==============================] - 84s 92ms/step - loss: 0.1163 - val_loss: 0.1131"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model.load_weights(\"weights/speaker_embedding_timeconv_wlstm_c2.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "  9/329 [..............................] - ETA: 39:47 - loss: 4.5454 - triplet_loss_output_loss: 0.4952 - targets_loss: 4.29 - ETA: 21:53 - loss: 4.1928 - triplet_loss_output_loss: 0.4471 - targets_loss: 3.96 - ETA: 15:57 - loss: 3.8784 - triplet_loss_output_loss: 0.4130 - targets_loss: 3.67 - ETA: 12:56 - loss: 3.6653 - triplet_loss_output_loss: 0.4223 - targets_loss: 3.45 - ETA: 11:07 - loss: 3.5140 - triplet_loss_output_loss: 0.4102 - targets_loss: 3.30 - ETA: 10:11 - loss: 3.3935 - triplet_loss_output_loss: 0.4261 - targets_loss: 3.18 - ETA: 9:36 - loss: 3.2792 - triplet_loss_output_loss: 0.4295 - targets_loss: 3.0644 - ETA: 9:02 - loss: 3.2052 - triplet_loss_output_loss: 0.4207 - targets_loss: 2.994 - ETA: 8:31 - loss: 3.1216 - triplet_loss_output_loss: 0.4200 - targets_loss: 2.9116"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-159-817f810c685d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      5\u001b[0m                               \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m                               \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m                               validation_data=([X_1,X_2,X_3],[y_triplet,y_target]))#valid_generator(batch_size),\n\u001b[0m\u001b[0;32m      8\u001b[0m                               \u001b[1;31m#validation_steps=int(np.ceil(valid_df.shape[0]//batch_size)))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[0;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 87\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m   2145\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[0;32m   2146\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2147\u001b[1;33m                                                class_weight=class_weight)\n\u001b[0m\u001b[0;32m   2148\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2149\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[1;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[0;32m   1837\u001b[0m             \u001b[0mins\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1838\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1839\u001b[1;33m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1840\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1841\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2355\u001b[0m         \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2356\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[1;32m-> 2357\u001b[1;33m                               **self.session_kwargs)\n\u001b[0m\u001b[0;32m   2358\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2359\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    887\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 889\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    890\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1118\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1120\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1121\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1315\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1317\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1318\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1319\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1321\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1322\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1323\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1324\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda2\\envs\\cx\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1300\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[0;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1302\u001b[1;33m                                    status, run_metadata)\n\u001b[0m\u001b[0;32m   1303\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1304\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "\n",
    "history = model.fit_generator(generator=train_generator(batch_size),\n",
    "                              steps_per_epoch=train_df.shape[0]//batch_size,\n",
    "                              epochs=100,\n",
    "                              callbacks=callbacks,\n",
    "                              validation_data=([X_1,X_2,X_3],[y_triplet,y_target]))#valid_generator(batch_size),\n",
    "                              #validation_steps=int(np.ceil(valid_df.shape[0]//batch_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding.save_weights(\"weights/speaker_embedding_t2.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.821782\n",
      "0.00257948\n",
      "0.97807\n",
      "-0.0011363\n",
      "0.990386\n",
      "0.00306028\n",
      "0.995768\n",
      "0.00208017\n",
      "1.00084\n",
      "-0.00353018\n",
      "1.10434\n",
      "0.0176663\n"
     ]
    }
   ],
   "source": [
    "for l in embedding.layers:\n",
    "    try:\n",
    "        print np.mean(l.get_weights()[0])\n",
    "    except:\n",
    "        continue\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.82218\n",
      "0.00224655\n",
      "0.978326\n",
      "-0.000955906\n",
      "0.990576\n",
      "0.00329765\n",
      "0.99577\n",
      "0.00218155\n",
      "1.00076\n",
      "-0.00361477\n",
      "1.10443\n",
      "0.0177416\n"
     ]
    }
   ],
   "source": [
    "for l in embedding.layers:\n",
    "    try:\n",
    "        print np.mean(l.get_weights()[0])\n",
    "    except:\n",
    "        continue\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n",
      "-0.000116561\n",
      "1.0\n",
      "-0.000564231\n",
      "1.0\n",
      "0.00077817\n",
      "1.0\n",
      "-6.0897e-05\n",
      "1.0\n",
      "0.000275884\n",
      "1.0\n",
      "-0.000213044\n"
     ]
    }
   ],
   "source": [
    "for l in embedding.layers:\n",
    "    try:\n",
    "        print np.mean(l.get_weights()[0])\n",
    "    except:\n",
    "        continue\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}