wandb resume

Niccolo-Ajroldi · Feb 26, 2024 · 0a9b713 · 0a9b713
1 parent 426ecee
commit 0a9b713
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 7 deletions.
diff --git a/algorithmic_efficiency/logger_utils.py b/algorithmic_efficiency/logger_utils.py
@@ -300,9 +300,13 @@ def __init__(self,
       self._tb_metric_writer = metric_writers.create_default_writer(events_dir)
       if wandb is not None and self.use_wandb:
         wandb.init(
-            dir=events_dir, tags=[flags.FLAGS.workload, flags.FLAGS.framework])
+            dir=events_dir, 
+            tags=[flags.FLAGS.workload, flags.FLAGS.framework],
+            resume=True, 
+            id="_".join(events_dir.split("/")[-4:])
+            )
         wandb.config.update(configs)
-        wandb.config.update(hyperparameters._asdict())
+        wandb.config.update(hyperparameters._asdict(), allow_val_change=True)
 
   def append_scalar_metrics(self,
                             metrics: Dict,

diff --git a/exp/seal/mnist.sh b/exp/seal/mnist.sh
@@ -19,8 +19,8 @@ workload=mnist
 dataset=MNIST
 submission='reference_algorithms/development_algorithms/mnist/mnist_pytorch/submission.py'
 search_space='reference_algorithms/development_algorithms/mnist/tuning_search_space.json'
-trials=2
-name="mnist_check/study_3"
+trials=1
+name="mnist_wandb_21/study_1"
 
 # Print GPU infos
 # nvidia-smi
@@ -37,6 +37,9 @@ python3 $CODE_DIR/submission_runner.py \
     --experiment_dir=$EXP_DIR  \
     --experiment_name=$name \
     --use_wandb \
-    --overwrite \
-    --max_global_steps 50 \
-    --save_checkpoints=False
+    --save_checkpoints=True \
+    --resume_last_run \
+    --rng_seed=1996
+
+    # --overwrite \
+    # --max_global_steps 1000 \