Merge pull request #735 from mlcommons/dev

Dev -> main

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Change Log
 
+## algoperf-benchmark-0.1.4 (2024-03-26)
+
+Upgrade CUDA version to CUDA 12.1:
+- Upgrade CUDA version in Dockerfiles that will be used for scoring.
+- Update Jax and PyTorch package version tags to use local CUDA installation.
+
+Add flag for completely disabling checkpointing. 
+- Note that we will run with checkpointing off at scoring time.
+
+Update Deepspeech and Conformer variant target setting configurations. 
+- Note that variant targets are not final. 
+
+Fixed bug in scoring code to take best trial in a study for external-tuning ruleset.
+
+Added instructions for submission. 
+
+Changed default number of workers for PyTorch data loaders to 0. Running with >0 may lead to incorrect eval results see https://github.com/mlcommons/algorithmic-efficiency/issues/732. 
+
 ## algoperf-benchmark-0.1.2 (2024-03-04)
 Workload variant additions and fixes:
 - Add Deepspeech workload variant

reference_algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json

@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.001308209823469072
+            0.0007852999990476642
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9731333693827139
+            0.6994142393023162
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9981232922116359
+            0.9918636824608852
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            9999
+            6000
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.16375311233774334
+            0.07286322158086678
         ]
     }
 }

reference_algorithms/target_setting_algorithms/librispeech_conformer_gelu/tuning_search_space.json

@@ -1,17 +1,17 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.001308209823469072
+            0.000590120167916659
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9731333693827139
+            0.737199286155609
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9981232922116359
+            0.05919391544031072
         ]
     },
     "warmup_steps": {
@@ -21,7 +21,7 @@
     },
     "weight_decay": {
         "feasible_points": [
-            0.16375311233774334
+            0.14128519778326312
         ]
     }
 }

reference_algorithms/target_setting_algorithms/librispeech_conformer_layernorm/tuning_search_space.json

@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.001308209823469072
+            0.0014446807792420305
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9731333693827139
+            0.7427148812902895
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9981232922116359
+            0.8993064520764248
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            9999
+            3000
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.16375311233774334
+            0.06875136511682291
         ]
     }
 }

reference_algorithms/target_setting_algorithms/librispeech_deepspeech_no_resnet/tuning_search_space.json

@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.0035278622506232458
+            0.0020162740358935045
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.8192305396005781
+            0.9604907112078142
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.495850879212151
+            0.8765457000160508
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            6000
+            3600
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.04339748256184769
+            0.0006149579248633481
         ]
     }
 }

reference_algorithms/target_setting_algorithms/librispeech_deepspeech_norm_and_spec_aug/tuning_search_space.json

@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.001308209823469072
+            0.0014446807792420305
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9731333693827139
+            0.7427148812902895
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9981232922116359
+            0.8993064520764248
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            6000
+            1800
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.16375311233774334
+            0.06875136511682291
         ]
     }
 }

reference_algorithms/target_setting_algorithms/librispeech_deepspeech_tanh/tuning_search_space.json

@@ -1,27 +1,27 @@
 {
     "learning_rate": {
         "feasible_points": [
-            0.003632312571224348
+            0.003604759885558324
         ]
     },
     "beta1": {
         "feasible_points": [
-            0.9980088784197237
+            0.9931094324430452
         ]
     },
     "beta2": {
         "feasible_points": [
-            0.9982275351621527
+            0.9976871843749077
         ]
     },
     "warmup_steps": {
         "feasible_points": [
-            6000
+            720
         ]
     },
     "weight_decay": {
         "feasible_points": [
-            0.2479797019098727
+            0.120077307855989
         ]
     }
 }

utils/workload_config.json → utils/target_setting_workload_config.json

@@ -176,7 +176,7 @@
     "librispeech_conformer_attention_temperature": {
         "max_steps": 80000,
         "dataset": "librispeech",
-        "submission_path": "reference_algorithms/target_setting_algorithms/jax_nadamw.py",
+        "submission_path": "reference_algorithms/target_setting_algorithms/jax_adamw.py",
         "tuning_search_space": "reference_algorithms/target_setting_algorithms/librispeech_conformer_attention_temperature/tuning_search_space.json"
     },
     "librispeech_conformer_gelu": {