-
Notifications
You must be signed in to change notification settings - Fork 241
/
run_burst.py
173 lines (146 loc) · 7.47 KB
/
run_burst.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
""" run_burst.py
The example commands given below expect the following folder structure:
- data
- gt
- burst
- {val,test}
- all_classes
- all_classes.json (filename is irrelevant)
- trackers
- burst
- exemplar_guided
- {val,test}
- my_tracking_method
- data
- results.json (filename is irrelevant)
- class_guided
- {val,test}
- my_other_tracking_method
- data
- results.json (filename is irrelevant)
Run example:
1) Exemplar-guided tasks (all three tasks share the same eval logic):
run_burst.py --USE_PARALLEL True --EXEMPLAR_GUIDED True --GT_FOLDER ../data/gt/burst/{val,test}/all_classes --TRACKERS_FOLDER ../data/trackers/burst/exemplar_guided/{val,test}
2) Class-guided tasks (common class and long-tail):
run_burst.py --USE_PARALLEL FTrue --EXEMPLAR_GUIDED False --GT_FOLDER ../data/gt/burst/{val,test}/all_classes --TRACKERS_FOLDER ../data/trackers/burst/class_guided/{val,test}
3) Refer to run_burst_ow.py for open world evaluation
Command Line Arguments: Defaults, # Comments
Eval arguments:
'USE_PARALLEL': False,
'NUM_PARALLEL_CORES': 8,
'BREAK_ON_ERROR': True,
'PRINT_RESULTS': True,
'PRINT_ONLY_COMBINED': False,
'PRINT_CONFIG': True,
'TIME_PROGRESS': True,
'OUTPUT_SUMMARY': True,
'OUTPUT_DETAILED': True,
'PLOT_CURVES': True,
Dataset arguments:
'GT_FOLDER': os.path.join(code_path, 'data/gt/burst/val'), # Location of GT data
'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/burst/class-guided/'), # Trackers location
'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER)
'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder)
'CLASSES_TO_EVAL': None, # Classes to eval (if None, all classes)
'SPLIT_TO_EVAL': 'training', # Valid: 'training', 'val'
'PRINT_CONFIG': True, # Whether to print current config
'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL
'MAX_DETECTIONS': 300, # Number of maximal allowed detections per image (0 for unlimited)
Metric arguments:
'METRICS': ['HOTA', 'CLEAR', 'Identity', 'TrackMAP']
"""
import sys
import os
import argparse
from tabulate import tabulate
from multiprocessing import freeze_support
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
import trackeval # noqa: E402
def main():
freeze_support()
# Command line interface:
default_eval_config = trackeval.Evaluator.get_default_eval_config()
default_eval_config['PRINT_ONLY_COMBINED'] = True
default_eval_config['DISPLAY_LESS_PROGRESS'] = True
default_eval_config['PLOT_CURVES'] = False
default_eval_config["OUTPUT_DETAILED"] = False
default_eval_config["PRINT_RESULTS"] = False
default_eval_config["OUTPUT_SUMMARY"] = False
default_dataset_config = trackeval.datasets.BURST.get_default_dataset_config()
# default_metrics_config = {'METRICS': ['HOTA', 'CLEAR', 'Identity', 'TrackMAP']}
# default_metrics_config = {'METRICS': ['HOTA']}
default_metrics_config = {'METRICS': ['HOTA', 'TrackMAP']}
config = {**default_eval_config, **default_dataset_config, **default_metrics_config} # Merge default configs
parser = argparse.ArgumentParser()
for setting in config.keys():
if type(config[setting]) == list or type(config[setting]) == type(None):
parser.add_argument("--" + setting, nargs='+')
else:
parser.add_argument("--" + setting)
args = parser.parse_args().__dict__
for setting in args.keys():
if args[setting] is not None:
if type(config[setting]) == type(True):
if args[setting] == 'True':
x = True
elif args[setting] == 'False':
x = False
else:
raise Exception('Command line parameter ' + setting + 'must be True or False')
elif type(config[setting]) == type(1):
x = int(args[setting])
elif type(args[setting]) == type(None):
x = None
else:
x = args[setting]
config[setting] = x
eval_config = {k: v for k, v in config.items() if k in default_eval_config.keys()}
dataset_config = {k: v for k, v in config.items() if k in default_dataset_config.keys()}
metrics_config = {k: v for k, v in config.items() if k in default_metrics_config.keys()}
# Run code
evaluator = trackeval.Evaluator(eval_config)
dataset_list = [trackeval.datasets.BURST(dataset_config)]
metrics_list = []
for metric in [trackeval.metrics.TrackMAP, trackeval.metrics.CLEAR, trackeval.metrics.Identity,
trackeval.metrics.HOTA]:
if metric.get_name() in metrics_config['METRICS']:
metrics_list.append(metric())
if len(metrics_list) == 0:
raise Exception('No metrics selected for evaluation')
output_res, output_msg = evaluator.evaluate(dataset_list, metrics_list, show_progressbar=True)
class_name_to_id = {x['name']: x['id'] for x in dataset_list[0].gt_data['categories']}
known_list = [4, 13, 1038, 544, 1057, 34, 35, 36, 41, 45, 58, 60, 579, 1091, 1097, 1099, 78, 79, 81, 91, 1115,
1117, 95, 1122, 99, 1132, 621, 1135, 625, 118, 1144, 126, 642, 1155, 133, 1162, 139, 154, 174, 185,
699, 1215, 714, 717, 1229, 211, 729, 221, 229, 747, 235, 237, 779, 276, 805, 299, 829, 852, 347,
371, 382, 896, 392, 926, 937, 428, 429, 961, 452, 979, 980, 982, 475, 480, 993, 1001, 502, 1018]
row_labels = ("HOTA", "DetA", "AssA", "AP")
trackers = list(output_res['BURST'].keys())
print("\n")
def average_metric(m):
return round(100*sum(m) / len(m), 2)
for tracker in trackers:
res = output_res['BURST'][tracker]['COMBINED_SEQ']
all_names = [x for x in res.keys() if (x != 'cls_comb_cls_av') and (x != 'cls_comb_det_av')]
class_split_names = {
"All": [x for x in res.keys() if (x != 'cls_comb_cls_av') and (x != 'cls_comb_det_av')],
"Common": [x for x in all_names if class_name_to_id[x] in known_list],
"Uncommon": [x for x in all_names if class_name_to_id[x] not in known_list]
}
# table columns: 'all', 'common', 'uncommon'
# table rows: HOTA, AssA, DetA, mAP
table_data = []
for row_label in row_labels:
row = [row_label]
for split_name in ["All", "Common", "Uncommon"]:
split_classes = class_split_names[split_name]
if row_label == "AP":
row.append(average_metric([res[c]['TrackMAP']["AP_all"].mean() for c in split_classes]))
else:
row.append(average_metric([res[c]['HOTA'][row_label].mean() for c in split_classes]))
table_data.append(row)
print(f"Results for Tracker: {tracker}\n")
print(tabulate(table_data, ["Metric", "All", "Common", "Uncommon"]))
if __name__ == '__main__':
main()