mc

script/lawa/condor.sub

@@ -2,7 +2,7 @@
 LOGS_DIR=/fast/najroldi/logs/algoperf
 EXE=/home/najroldi/algorithmic-efficiency/script/lawa/auto_run.sh
 
-workload=criteo1tb
+workload=wmt
 framework=pytorch
 
 # ## NadamW
@@ -17,10 +17,10 @@ framework=pytorch
 
 ## LAWA
 submission=submissions/lawa_ema/lawa_ema.py
-search_space=script/lawa/ema/lawa_trial_5_tune_04.json
+search_space=script/lawa/ema/lawa_trial_5_best.json
 num_tuning_trials=12
-study=3
-name=lawa_ema_tune_06
+study=1
+name=lawa_ema_tune_08
 rng_seed=$(study)
 allow_tf_32=1
 num_jobs=$(num_tuning_trials)
@@ -49,8 +49,8 @@ log = $(LOGS_DIR)/log/job.$(Cluster).$(Process).log
 request_memory = 500000
 request_cpus = 36
 request_gpus = 4
-requirements = (TARGET.CUDADeviceName == "NVIDIA A100-SXM4-40GB")
-# requirements = (TARGET.CUDADeviceName == "NVIDIA A100-SXM4-80GB")
+# requirements = (TARGET.CUDADeviceName == "NVIDIA A100-SXM4-40GB")
+requirements = (TARGET.CUDADeviceName == "NVIDIA A100-SXM4-80GB")
 # requirements = (TARGET.CUDACapability >= 8.0)
 
 queue $(num_jobs)

script/lawa/ema/lawa_trial_3_tune_01.json

@@ -0,0 +1,32 @@
+{
+  "dropout_rate": {
+    "feasible_points": [0.0]
+  },
+  "label_smoothing": {
+    "feasible_points": [0.0]
+  },
+  "learning_rate": {
+    "feasible_points": [0.001308209823469072]
+  },
+  "one_minus_beta1": {
+    "feasible_points": [0.02686663061]
+  },
+  "beta2": {
+    "feasible_points": [0.9981232922116359]
+  },
+  "weight_decay": {
+    "feasible_points": [0.16375311233774334]
+  },
+  "warmup_factor": {
+    "feasible_points": [0.1]
+  },
+  "lawa_burnin_pct": {
+    "feasible_points": [0.0]
+  },
+  "lawa_every_pct": {
+    "feasible_points": [0.0002, 0.002, 0.02]
+  },
+  "lawa_beta": {
+    "feasible_points": [0.2, 0.6, 0.8, 0.9]
+  }
+}

script/lawa/ema/lawa_trial_5_tune_04 copy.json → script/lawa/ema/lawa_trial_5_best.json

@@ -21,12 +21,12 @@
     "feasible_points": [0.02]
   },
   "lawa_burnin_pct": {
-    "feasible_points": [0.0, 0.1, 0.5]
+    "feasible_points": [0.0]
   },
   "lawa_every_pct": {
     "feasible_points": [0.002]
   },
   "lawa_beta": {
-    "feasible_points": [0.2, 0.6, 0.8, 0.9]
+    "feasible_points": [0.8]
   }
 }

script/lawa/ema/lawa_trial_5_tune_05.json

@@ -0,0 +1,32 @@
+{
+  "dropout_rate": {
+    "feasible_points": [0.1]
+  },
+  "label_smoothing": {
+    "feasible_points": [0.0]
+  },
+  "learning_rate": {
+    "feasible_points": [0.00175]
+  },
+  "one_minus_beta1": {
+    "feasible_points": [0.06733926164]
+  },
+  "beta2": {
+    "feasible_points": [0.9955159689799007]
+  },
+  "weight_decay": {
+    "feasible_points": [0.08121616522670176]
+  },
+  "warmup_factor": {
+    "feasible_points": [0.02]
+  },
+  "lawa_burnin_pct": {
+    "feasible_points": [0.0, 0.1, 0.5, 0.7]
+  },
+  "lawa_every_pct": {
+    "feasible_points": [0.002]
+  },
+  "lawa_beta": {
+    "feasible_points": [0.95]
+  }
+}