v2 is working as expected.

CustomNets.py

@@ -33,10 +33,10 @@ def build(self, input_shape):
         super(MyLayer, self).build(input_shape)  # Be sure to call this at the end
 
     def call(self, x):
-        return K.dot(x, self.kernel)
+        return [K.dot(x, self.kernel), K.dot(x, self.kernel)]
 
     def compute_output_shape(self, input_shape):
-        return (input_shape[0], self.output_dim)
+        return [(input_shape[0], self.output_dim), (input_shape[0], self.output_dim)]
 
 
 class NetVLADLayer( Layer ):
@@ -68,8 +68,8 @@ def call( self, x ):
         # soft-assignment.
         s = K.conv2d( x, self.kernel, padding='same' ) + self.bias
         a = K.softmax( s )
-        # self.amap = K.argmax( a, -1 )
-        # print 'amap.shape', self.amap.shape
+        self.amap = K.argmax( a, -1 )
+        print 'amap.shape', self.amap.shape
 
         # Dims used hereafter: batch, H, W, desc_coeff, cluster
         a = K.expand_dims( a, -2 )
@@ -91,34 +91,37 @@ def call( self, x ):
         v = K.batch_flatten( v )
         v = K.l2_normalize( v, axis=-1 )
 
-        return v
+        return [v, self.amap]
 
     def compute_output_shape( self, input_shape ):
         # return (input_shape[0], self.v.shape[-1].value )
-        return (input_shape[0], self.K*self.D )
+        # return [(input_shape[0], self.K*self.D ), (input_shape[0], self.amap.shape[1].value, self.amap.shape[2].value) ]
+        return [(input_shape[0], self.K*self.D ), (input_shape[0], input_shape[1], input_shape[2]) ]
 
 
 def make_vgg( input_img ):
+    r_l2=keras.regularizers.l2(0.01)
+    r_l1=keras.regularizers.l1(0.01)
 
-    x_64 = keras.layers.Conv2D( 64, (3,3), padding='same', activation='relu' )( input_img )
-    # BN
-    x_64 = keras.layers.Conv2D( 64, (3,3), padding='same', activation='relu' )( x_64 )
-    # BN
+    x_64 = keras.layers.Conv2D( 64, (3,3), padding='same', activation='relu', kernel_regularizer=r_l2, activity_regularizer=r_l1 )( input_img )
+    x_64 = keras.layers.normalization.BatchNormalization()( x_64 )
+    x_64 = keras.layers.Conv2D( 64, (3,3), padding='same', activation='relu', kernel_regularizer=r_l2, activity_regularizer=r_l1 )( x_64 )
+    x_64 = keras.layers.normalization.BatchNormalization()( x_64 )
     x_64 = keras.layers.MaxPooling2D( pool_size=(2,2), padding='same' )( x_64 )
 
 
-    x_128 = keras.layers.Conv2D( 128, (3,3), padding='same', activation='relu' )( x_64 )
-    # BN
-    x_128 = keras.layers.Conv2D( 128, (3,3), padding='same', activation='relu' )( x_128 )
-    # BN
+    x_128 = keras.layers.Conv2D( 128, (3,3), padding='same', activation='relu', kernel_regularizer=r_l2, activity_regularizer=r_l1 )( x_64 )
+    x_128 = keras.layers.normalization.BatchNormalization()( x_128 )
+    x_128 = keras.layers.Conv2D( 128, (3,3), padding='same', activation='relu', kernel_regularizer=r_l2, activity_regularizer=r_l1 )( x_128 )
+    x_128 = keras.layers.normalization.BatchNormalization()( x_128 )
     x_128 = keras.layers.MaxPooling2D( pool_size=(2,2), padding='same' )( x_128 )
 
 
-    x_256 = keras.layers.Conv2D( 256, (3,3), padding='same', activation='relu' )( x_128 )
-    # # BN
-    x_256 = keras.layers.Conv2D( 256, (3,3), padding='same', activation='relu' )( x_256 )
-    # # BN
-    x_256 = keras.layers.MaxPooling2D( pool_size=(2,2), padding='same' )( x_256 )
+    # x_256 = keras.layers.Conv2D( 256, (3,3), padding='same', activation='relu' )( x_128 )
+    # x_256 = keras.layers.normalization.BatchNormalization()( x_256 )
+    # x_256 = keras.layers.Conv2D( 256, (3,3), padding='same', activation='relu' )( x_256 )
+    # x_256 = keras.layers.normalization.BatchNormalization()( x_256 )
+    # x_256 = keras.layers.MaxPooling2D( pool_size=(2,2), padding='same' )( x_256 )
 
     #
     # x_512 = keras.layers.Conv2D( 512, (3,3), padding='same', activation='relu' )( x_256 )

TimeMachineRender.py

@@ -216,7 +216,7 @@ def _different(self, n, loc_idx, yr_idx, im_idx ):
     ## Given a set for example, L = [(25, 0, 7), (25, 0, 5), (25, 0, 6), (25, 0, 4), (25, 0, 4)]
     ## Load corresponding images. Returns a np.array of size nx240x320x3
     ## If apply_distortions is true, random distortions will be applied. Currently planar rotations with angles as Gaussian distribution centered at 0, sigma=25
-    def _get_images(self, L, apply_distortions=False, return_gray=False, PRINTING=False):
+    def _get_images(self, L, resize=None, apply_distortions=False, return_gray=False, PRINTING=False):
         pyDB = self.pyDB
         A = []
         for loc_idx, yr_idx, im_idx in L:
@@ -232,7 +232,10 @@ def _get_images(self, L, apply_distortions=False, return_gray=False, PRINTING=Fa
                 if PRINTING:
                     print 'imread : ', file_name
                 # TODO blur before resizing
-                IM = cv2.resize( cv2.imread( file_name ) , (320,240)  )
+                if resize is None:
+                    IM = cv2.imread( file_name )
+                else:
+                    IM = cv2.resize( cv2.imread( file_name ) , resize  )
                 # IM = cv2.resize( cv2.imread( file_name ) , (160,120)  )
             except:
                 print 'im_indx error', im_list
@@ -270,7 +273,8 @@ def _get_images(self, L, apply_distortions=False, return_gray=False, PRINTING=Fa
                 IM = np.expand_dims( IM_gray, axis=2 )
 
 
-            A.append( IM[:,:,::-1] )
+            # A.append( IM[:,:,::-1] )
+            A.append( IM )
 
         return np.array(A)
         #cv2.imshow( 'win', np.concatenate( A, axis=1 )  )
@@ -282,7 +286,7 @@ def _get_images(self, L, apply_distortions=False, return_gray=False, PRINTING=Fa
     # Gives out `nP` number of positive samples of query image. `nN` number of negative samples.
     # Note, query image is the 0th image. Next nP will be positive, next nN will be negative.
     # return_gray=True will return a (N,240,320,1), ie gray scale images
-    def step(self, nP, nN, return_gray=False, ENABLE_IMSHOW=False):
+    def step(self, nP, nN, resize=None, apply_distortions=False, return_gray=False, ENABLE_IMSHOW=False):
         # np.random.seed(1)
         # Will generate a total of 1+nP+nN number of images. 1st is the query image (choosen randomly)
         # Next nP will be positive Samples. Next nN will be negative samples
@@ -297,9 +301,9 @@ def step(self, nP, nN, return_gray=False, ENABLE_IMSHOW=False):
         # print 'd : ', diffs
 
         PRINTING = False
-        q_im = self._get_images( [(loc_idx, yr_idx, im_idx)], return_gray=return_gray , PRINTING=PRINTING )
-        sims_im = self._get_images(sims[0:nP], apply_distortions=True, return_gray=return_gray, PRINTING=PRINTING)
-        diffs_im = self._get_images(diffs, return_gray=return_gray, PRINTING=PRINTING)
+        q_im = self._get_images( [(loc_idx, yr_idx, im_idx)], resize=resize, apply_distortions=apply_distortions, return_gray=return_gray , PRINTING=PRINTING )
+        sims_im = self._get_images(sims[0:nP], resize=resize, apply_distortions=apply_distortions, return_gray=return_gray, PRINTING=PRINTING)
+        diffs_im = self._get_images(diffs, resize=resize, apply_distortions=apply_distortions, return_gray=return_gray, PRINTING=PRINTING)
 
 
         # print q_im.shape

WalksRenderer.py

@@ -39,6 +39,7 @@ def __init__( self, db_path ):
             cap = cv2.VideoCapture( file_name )
 
             if cap.isOpened():
+
                 nFrames = cap.get( cv2.CAP_PROP_FRAME_COUNT )
                 print tcolor.OKBLUE, '+    %03d nFrames=%06d' %(_i, nFrames), file_name, tcolor.ENDC
                 self.captures.append( (cap, nFrames) )

noveou_train_netvlad.py

@@ -29,7 +29,7 @@
 from PittsburgRenderer import PittsburgRenderer
 
 
-if __name__ == '__main__': # Testing renderers
+if __name__ == '__1main__': # Testing renderers
     nP = 2
     nN = 2
 
@@ -159,8 +159,8 @@
 
 
 
-
-if __name__ == '__1main__':
+from CustomNets import MyLayer
+if __name__ == '__main__':
     input_img = keras.layers.Input( shape=(256,) )
 
     out = MyLayer( 15 )( input_img )

noveou_train_netvlad_v2.py

@@ -28,6 +28,7 @@
 
 
 def custom_loss(y_true, y_pred):
+    """ All pair loss """
     nP = 2
     nN = 2
 
@@ -51,17 +52,51 @@ def custom_loss(y_true, y_pred):
     return K.maximum(zeros, aux)
 
 
+def custom_metric( y_true, y_pred ):
+    nP = 2
+    nN = 2
+
+    # y_pred.shape = shape=(?, 5, 512)
+    q = y_pred[:,0:1,:]    # shape=(?, 1, 512)
+    P = y_pred[:,1:1+nP,:] # shape=(?, 2, 512)
+    N = y_pred[:,1+nP:,:]  # shape=(?, 2, 512)
+    q_dot_P = keras.layers.dot( [q,P], axes=-1 )  # shape=(?, 1, 2)
+    q_dot_N = keras.layers.dot( [q,N], axes=-1 )  # shape=(?, 1, 2)
+
+    epsilon = 0.0#0.1  # Your epsilon here
+
+    zeros = K.zeros((nP, nN), dtype='float32')
+    ones_m = K.ones(nP, dtype='float32')
+    ones_n = K.ones(nN, dtype='float32')
+    # code.interact( local=locals() , banner='custom_loss')
+    aux = ones_m[None, :, None] * q_dot_N[:, None, :] \
+          - q_dot_P[:, :, None] * ones_n[None, None, :] \
+          + epsilon * ones_m[:, None] * ones_n[None, :]
+
+    return K.min(q_dot_P) - q_dot_N
+
+    Q = K.less(zeros, aux)
+    K.print_tensor( Q, message='Q : \n')
+    r = K.sum( K.cast(Q, 'float32' ) )
+    code.interact( local=locals() )
+    return r
+
+
+
 if __name__ == '__main__':
     nP = 2
     nN = 2
 
     #------------------------------------------------------------------------
     # Load data on RAM
     #------------------------------------------------------------------------
-    PTS_BASE = '/Bulk_Data/data_Akihiko_Torii/Pitssburg/'
-    pr = PittsburgRenderer( PTS_BASE )
+    # PTS_BASE = '/Bulk_Data/data_Akihiko_Torii/Pitssburg/'
+    # pr = PittsburgRenderer( PTS_BASE )
+
+    TTM_BASE = '/Bulk_Data/data_Akihiko_Torii/Tokyo_TM/tokyoTimeMachine/' #Path of Tokyo_TM
+    pr = TimeMachineRender( TTM_BASE )
     D = []
-    for s in range(100):
+    for s in range(50):
         print 'get a sample #', s
         a,_ = pr.step(nP=nP, nN=nN, return_gray=False, resize=(240,320), apply_distortions=False, ENABLE_IMSHOW=False)
         print a.shape
@@ -87,14 +122,16 @@ def custom_loss(y_true, y_pred):
     # cnn = input_img
     cnn = make_vgg( input_img )
 
-    out = NetVLADLayer(num_clusters = 16)( cnn )
+    out, out_amap = NetVLADLayer(num_clusters = 16)( cnn )
     # code.interact( local=locals() )
     model = keras.models.Model( inputs=input_img, outputs=out )
 
     model.summary()
     keras.utils.plot_model( model, to_file='core.png', show_shapes=True )
 
 
+
+
     #--------------------------------------------------------------------------
     # TimeDistributed
     #--------------------------------------------------------------------------
@@ -107,10 +144,23 @@ def custom_loss(y_true, y_pred):
     keras.utils.plot_model( t_model, to_file='core_t.png', show_shapes=True )
 
 
+    # parallel_t_model = keras.utils.multi_gpu_model(t_model, gpus=2)
+
+
     #--------------------------------------------------------------------------
     # Compile
     #--------------------------------------------------------------------------
-    t_model.compile( loss=custom_loss, optimizer='sgd' )
+    rmsprop = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.01, clipnorm=1.0)
+    t_model.compile( loss=custom_loss, optimizer=rmsprop, metrics=[custom_loss] )
+
+    # TODO: callbacks : Tensorboard, lr, multigpu
+    # TODO: Validation split and custom validation function
+    import tensorflow as tf
+    tb = tf.keras.callbacks.TensorBoard( log_dir='tensorboard.logs/noveou' )
+    reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=0.001)
 
     t_model.fit( x=D, y=np.zeros(D.shape[0]),
-                epochs=5, batch_size=20, verbose=2 )
+                epochs=5, batch_size=3, verbose=1, validation_split=0.1,
+                callbacks=[tb] )
+
+    model.save( 'model.keras/core_model_tokyotm.keras')

test_keras_model.py

@@ -8,6 +8,7 @@
 import keras
 import code
 import numpy as np
+import time
 
 import cv2
 
@@ -22,19 +23,35 @@
 from WalksRenderer import WalksRenderer
 from PittsburgRenderer import PittsburgRenderer
 
+# Custom Utils
+from ColorLUT import ColorLUT
+colx = ColorLUT()
 
 if __name__ == '__main__':
-    nP = 2
-    nN = 2
+    # WR_BASE = '/Bulk_Data/keezi_walks/'
+    WR_BASE = '/media/mpkuse/Bulk_Data/keezi_walks/'
+    wr = WalksRenderer( WR_BASE )
+    for i in range(20):
+        a,b = wr.step(nP=10, nN=10)
+        print a.shape
+        print b.shape
+        cv2.waitKey(0)
+
+if __name__ == '__1main__':
+    nP = 3
+    nN = 1
 
     #------
     # Load data on RAM
     #------
 
-    PTS_BASE = '/Bulk_Data/data_Akihiko_Torii/Pitssburg/'
-    pr = PittsburgRenderer( PTS_BASE )
+    # PTS_BASE = '/Bulk_Data/data_Akihiko_Torii/Pitssburg/'
+    # pr = PittsburgRenderer( PTS_BASE )
+
+    TTM_BASE = '/Bulk_Data/data_Akihiko_Torii/Tokyo_TM/tokyoTimeMachine/' #Path of Tokyo_TM
+    pr = TimeMachineRender( TTM_BASE )
 
-    a, _ = pr.step( nP=5, nN=5, return_gray=False, resize=None, apply_distortions=False, ENABLE_IMSHOW=False )
+    a, _ = pr.step( nP=nP, nN=nN, return_gray=False, resize=None, apply_distortions=False, ENABLE_IMSHOW=False )
 
     input_img = keras.layers.Input( shape=(480, 640, 3 ) )
     # cnn = input_img
@@ -44,7 +61,20 @@
     # code.interact( local=locals() )
     model = keras.models.Model( inputs=input_img, outputs=out )
 
-    model.load_weights( 'model.keras/core_model.keras' )
+    # model.load_weights( 'model.keras/core_model_tryh.keras' )
+    model.load_weights( 'model.keras/core_model_tryh.keras' )
+    # model.load_weights( 'model.keras/core_model_tokyotm.keras' )
+
+    # quit()
+    start_t = time.time()
+    for x in range(a.shape[0]):
+        out, out_amap = model.predict( a[x:x+1,:,:,:] )
+        out_amap_lut = colx.lut( out_amap[0,:,:] )
+        cv2.imshow( 'im', a[x,:,:,:].astype('uint8'))
+        cv2.imshow( 'out_amap_lut', out_amap_lut )
+        cv2.waitKey(0)
 
-    out = model.predict( a[:,:,:,:] )
-    np.matmul( out[0:1,:], np.transpose( out[1:,:] ) )
+    print 'predicted in %4.2fms' %( 1000. * (time.time() - start_t))
+    out, out_amap = model.predict( a[:,:,:,:] )
+    M = np.matmul( out[0:1,:], np.transpose( out[1:,:] ) )
+    print M

test_render.py

@@ -59,6 +59,8 @@ def demo_walks():
         preloading image frames of the videos.
     """
     WR_BASE = './keezi_walks/'
+    WR_BASE = '/media/mpkuse/Bulk_Data/keezi_walks/'
+
     wr = WalksRenderer( WR_BASE )
     for i in range(20):
         a,b = wr.step(nP=10, nN=10)
@@ -90,7 +92,7 @@ def demo_panda():
     for i in range(20):
         a,b = app.step(16)
 
-demo_pittsburg()
-# demo_walks()
+# demo_pittsburg()
+demo_walks()
 # demo_tokyotm()
 # demo_panda()