first commit

.gitignore

@@ -0,0 +1,119 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Dataset
+CityScapes/
+Lane/
+
+# Generated
+scores/
+
+# tmp files
+*.pyc
+runs/
+*.pickle
+*.pth
+saving/*
+*.swp

README.md

@@ -0,0 +1,72 @@
+# Object Descriptor
+
+This repo contains source codes for the arXiv preprint ["A Robust Object Encoding Method"](https://arxiv.org/abs/2105.00327)
+
+
+## Demo
+Object matching comparison when the objects are non-rigid and the view is changed, left is the result of our method while right is the result of NetVLAD
+
+![](experiments/demo/object-matching1.gif) ![](experiments/demo/object-matching2.gif) 
+
+Relocalization in KITTI
+
+![](experiments/demo/kitti-relocalization.gif)
+
+
+## Dependencies
+* Python
+* PyTorch
+* OpenCV
+* Matplotlib
+* NumPy
+* Yaml
+
+
+## Data
+Four datasets are used in our experiments.
+
+### KITTI Odometry
+For relocalization experiment. Three sequences are selected, and they are "00", "05" and "06".
+
+### KITTI Tracking
+For multi-object matching experiment. Four sequences are selected, and they are "0002", "0003", "0006", "0010".
+
+### VOT Datasets
+For single-object matching experiment. We select three sequences from VOT2019 datasets and they are "bluecar", "bus6" and "humans_corridor_occ_2_A", because the tracked objects in these sequences are included in coco datasets, which are the data we used to train mask-rcnn. 
+
+### OTB Datasets
+For single-object matching experiment. We select five sequences and they are "BlurBody", "BlurCar2", "Human2", "Human7" and "Liquor".
+
+
+## Examples
+
+### Relocalization in KITTI Datasets
+
+1. Extract object descrptors
+   ```
+   python experiments/place_recogination/online_relocalization.py -c config/experiment_tracking.yaml -g 1 -s PATH_TO_SAVE_MIDDLE_RESULTS -d PATH_TO_DATASET -m PATH_TO_MODELS
+   ```
+
+2. Compute precision-recall curves
+   ```
+   python experiments/place_recogination/offline_process.py -c config/experiment_tracking.yaml -g 1 -d PATH_TO_DATASET -n PATH_TO_MIDDLE_RESULTS -s PATH_TO_SAVE_RESULTS
+   ```
+
+3. Compute top-K relocalization results
+   ```
+   python experiments/place_recogination/offline_topK.py -c config/experiment_tracking.yaml -g 1 -d PATH_TO_DATASET -n PATH_TO_MIDDLE_RESULTS -s PATH_TO_SAVE_RESULTS
+   ```
+
+### Object Matching in OTB, VOT or KITTI Tracking Datasets
+
+* Run multi-object matching experiment in KITTI Tracking Datasets
+  Modify the [config file](config/experiment_tracking.yaml) and run  
+  ```
+  python experiments/object_tracking/object_tracking.py -c config/experiment_tracking.yaml -g 1 -s PATH_TO_SAVE_RESULTS -d PATH_TO_DATASET -m PATH_TO_MODELS 
+  ```
+
+* Run single-object matching experiment in OTB or VOT Datasets
+  Modify the [config file](config/experiment_tracking.yaml) and run  
+  ```
+  python experiments/object_tracking/single_object_tracking.py -c config/experiment_tracking.yaml -g 1 -s PATH_TO_SAVE_RESULTS -d PATH_TO_DATASET -m PATH_TO_MODELS 
+  ```

config/compare_tracking.yaml

@@ -0,0 +1,9 @@
+intervals: [1, 3, 5]  # select lines, the number represent the interval
+title: kitti          # figure title
+colors: ['green', 'red', 'blue', 'yellow', 'darkviolet', 'sandybrown']  # lines colors
+linewidth: 3          # line width
+xlabel: recall        # x-axis name 
+ylabel: precision     # y-axis name
+fontsize: 20          # font size
+figsize: (10, 10)     # fingure size, inch
+dpi: 100              # dots per inch

config/experiment_place_recognition.yaml

@@ -0,0 +1,25 @@
+data:
+  name: 'kitti'
+  nclass: 81
+  normal_size: [384, 1280] # min_size, max_size
+  cache_in_memory: false
+  validation_size: 96
+model:
+  superpoint:
+    cell : 8
+    detection_threshold: 0.2
+  maskrcnn:
+    add_maskrcnn: true
+    trainable_layers: 5  # backbone trainable layers
+    fix_backbone: true
+    backbone_type: 'resnet50'
+    image_mean: [0.45, 0.45, 0.45]
+    image_std: [0.225, 0.225, 0.225]
+  gcn:
+    descriptor_dim: 256
+    points_encoder_dims: [2, 4, 8, 16]
+    hidden_dim: 512
+    dropout: 0
+    alpha: 0.2
+    nheads: 4
+    nout: 2048

config/experiment_tracking.yaml

@@ -0,0 +1,30 @@
+data:
+  ### kitti datasets
+  name: 'kitti'
+  normal_size: [384, 1280] # min_size, max_size
+  ### OTB or VOT datasets
+  # name: 'otb'
+  # normal_size: [384, 1280] # min_size, max_size  
+  ###
+  nclass: 81
+  cache_in_memory: false
+  validation_size: 96
+model:
+  superpoint:
+    cell : 8
+    detection_threshold: 0.2
+  maskrcnn:
+    add_maskrcnn: true
+    trainable_layers: 5  # backbone trainable layers
+    fix_backbone: true
+    backbone_type: 'resnet50'
+    image_mean: [0.45, 0.45, 0.45]
+    image_std: [0.225, 0.225, 0.225]
+  gcn:
+    descriptor_dim: 256
+    points_encoder_dims: [2, 4, 8, 16]
+    hidden_dim: 512
+    dropout: 0
+    alpha: 0.2
+    nheads: 4
+    nout: 2048

config/train_gcn_coco.yaml

@@ -0,0 +1,90 @@
+data:
+  name: 'coco'
+  nclass: 81
+  # TRAIN: 'coco_2014_train'
+  TRAIN: 'coco_2014_train'
+  VAL: 'coco_2014_minival'
+  TEST: 'coco_2014_valminusminival'
+  normal_size: [320, 320] # min_size, max_size
+  cache_in_memory: false
+  validation_size: 96
+  augmentation:
+    photometric:
+      enable: true
+      primitives: [
+          'random_brightness', 'random_contrast', 'additive_speckle_noise',
+          'additive_gaussian_noise', 'add_shade', 'motion_blur']
+      params:
+        random_brightness: {max_change: 50}
+        random_contrast: {max_change: [0.5, 1.5]}
+        additive_gaussian_noise: {std: [0, 10]}
+        additive_speckle_noise: {intensity: 5}
+        add_shade:
+          amplitude: [-0.5, 0.5]
+          kernel_size_interval: [100, 150]
+        motion_blur: {max_ksize: 3}
+    homographic:
+      enable: false  # not implemented
+    gcn_mask:
+      enable: false
+      primitives: [
+          'erode', 'dilate', 'random_region_zero', 'random_block_zero', 
+          'random_block_one']
+      params:
+        erode: {kernel_size: 10}
+        dilate: {kernel_size: 10}
+        random_region_zero: 
+          scale_x: 0.3
+          scale_y: 0.3
+        random_block_zero: 
+          num: 5
+          size: 10
+        random_block_one:
+          num: 5
+          size: 10
+  warped_pair:
+    enable: false
+    params:
+      translation: true
+      rotation: true
+      scaling: true
+      perspective: true
+      scaling_amplitude: 0.2
+      perspective_amplitude_x: 0.2
+      perspective_amplitude_y: 0.2
+      patch_ratio: 0.85
+      max_angle: 1.57
+      allow_artifacts: true
+    valid_border_margin: 3
+model:
+  superpoint:
+    cell : 8
+    detection_threshold: 0.2
+  mask_rcnn:
+    add_maskrcnn: true
+    trainable_layers: 5  # backbone trainable layers
+    fix_backbone: true
+    backbone_type: 'resnet50'
+    image_mean: [0.45, 0.45, 0.45]
+    image_std: [0.225, 0.225, 0.225]
+  gcn:
+    descriptor_dim: 256
+    points_encoder_dims: [2, 4, 8, 16]
+    hidden_dim: 512
+    dropout: 0
+    alpha: 0.2
+    nheads: 4
+    nout: 2048
+    train:
+      batch_szie: 16
+      positive_margin: 1
+      negative_margin: 0.2
+      lambda_d: 1
+      epochs     : 100
+      lr         : 0.000001
+      momentum   : 0
+      w_decay    : 0.0001
+      milestones : [1000, 2000, 5000, 10000, 15000] # iter
+      gamma      : 0.3
+      checkpoint: 1000
+      weight_lambda: [0.1, 0.1]

config/train_maskrcnn_coco.yaml

@@ -0,0 +1,25 @@
+data:
+  name: 'coco'
+  nclass: 81
+  # TRAIN: 'coco_2014_train'
+  TRAIN: 'coco_2014_train'
+  VAL: 'coco_2014_minival'
+  TEST: 'coco_2014_valminusminival'
+  normal_size: [320, 320] # min_size, max_size
+  cache_in_memory: false
+  validation_size: 96
+model:
+  maskrcnn: 
+    trainable_layers: 5  # backbone trainable layers
+    backbone_type: 'resnet50'
+    image_mean: [0.45, 0.45, 0.45]
+    image_std: [0.225, 0.225, 0.225]
+    batch_size : 8
+    epochs     : 10
+    lr         : 0.00001
+    momentum   : 0
+    w_decay    : 0.0001
+    milestones : [10000, 20000, 50000, 100000, 150000] # iter
+    gamma      : 0.3
+    dataset_size : 10000
+    checkpoint: 1000