-
Notifications
You must be signed in to change notification settings - Fork 0
/
analyze_budget.py
104 lines (86 loc) · 3.92 KB
/
analyze_budget.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
'''
File: /analyze.py
Project: learning-hive
Created Date: Monday March 20th 2023
Author: Long Le (vlongle@seas.upenn.edu)
Copyright (c) 2023 Long Le
'''
'''
File: /plot.py
Project: lifelong-learning-viral
Created Date: Wednesday March 15th 2023
Author: Long Le (vlongle@seas.upenn.edu)
Copyright (c) 2023 Long Le
'''
"""
For cifar100, epochs=500 is stored in
"""
import os
import re
from shell.utils.metric import Metric
from shell.utils.record import Record
root_result_dir = "budget_experiment_results/jorge_setting_recv_variable_shared_memory_size"
# root_result_dir = "combined_recv_remove_neighbors_results"
# root_result_dir = "topology_experiment_results/modmod"
# root_result_dir = "topology_experiment_results/topology_experiment_results/jorge_setting_fedavg/comm_freq_5"
# root_result_dir = "budget_experiment_results/jorge_setting_fedavg"
# root_result_dir = "budget_experiment_results/modmod"
record = Record(f"{root_result_dir}.csv")
pattern = r".*"
num_init_tasks = 4 # vanilla_results
for result_dir in os.listdir(root_result_dir):
for job_name in os.listdir(os.path.join(root_result_dir, result_dir)):
use_contrastive = "contrastive" in job_name
for dataset_name in os.listdir(os.path.join(root_result_dir, result_dir, job_name)):
for algo in os.listdir(os.path.join(root_result_dir, result_dir, job_name, dataset_name)):
for seed in os.listdir(os.path.join(root_result_dir, result_dir, job_name, dataset_name, algo)):
for agent_id in os.listdir(os.path.join(root_result_dir, result_dir, job_name, dataset_name, algo, seed)):
if agent_id == "hydra_out" or agent_id == "agent_69420":
continue
save_dir = os.path.join(root_result_dir,
result_dir, job_name, dataset_name, algo, seed, agent_id)
# if the pattern doesn't match, continue
if not re.search(pattern, save_dir):
continue
m = Metric(save_dir, num_init_tasks)
# extra_algo = f"{result_dir}_{algo}"
extra_algo = f"{algo}_{result_dir}"
# print(save_dir, 'algo', extra_algo)
record.write(
{
"dataset": dataset_name,
"algo": extra_algo,
"use_contrastive": use_contrastive,
"seed": seed,
"agent_id": agent_id,
"avg_acc": m.compute_avg_accuracy(),
"final_acc": m.compute_final_accuracy(),
"auc": m.compute_auc(),
}
)
# print('record', record.df)
# exit(0)
print(record.df)
# get the final accuracy with respect to different algo and dataset
# and whether it uses contrastive loss
print("=====FINAL ACC======")
print(record.df.groupby(["algo", "dataset", "use_contrastive"])[
"final_acc"].mean() * 100)
# print(record.df.groupby(["algo", "dataset", "use_contrastive"])[
# "final_acc"].sem() * 100)
print("=====AUC======")
print(record.df.groupby(["algo", "dataset", "use_contrastive"])[
"auc"].mean())
# print("=====AVG ACC======")
# print(record.df.groupby(["algo", "dataset", "use_contrastive"])[
# "avg_acc"].mean() * 100)
# # print("=====BACKWARD======")
# # print(record.df.groupby(["algo", "dataset", "use_contrastive"])[
# # "backward"].mean())
# print("=====FORWARD======")
# print(record.df.groupby(["algo", "dataset", "use_contrastive"])[
# "forward"].mean() * 100)
# print("=====CATASTROPHIC======")
# print(record.df.groupby(["algo", "dataset", "use_contrastive"])[
# "catastrophic"].mean())
record.save()