Add an accuracy column for open submissions

mlcommons · Mar 23, 2022 · ca18f5c · ca18f5c
1 parent 03596fb
commit ca18f5c
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/tools/submission/generate-final-report.py b/tools/submission/generate-final-report.py
@@ -81,6 +81,7 @@ def main():
   indices['open'] = indices['closed'].copy()
   indices['closed'].append('Details')
   indices['open'].append('UsedModel')
+  indices['open'].append('Accuracy')
   indices['open'].append('Details')
   columns = [
       'Model',

diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py
@@ -612,13 +612,20 @@
 }
 
 ACC_PATTERN = {
-    "acc": r"^accuracy=([\d\.]+).*",
-    "AUC": r"^AUC=([\d\.]+).*",
-    "mAP": r"^mAP=([\d\.]+).*",
-    "bleu": r"^BLEU\:\s*([\d\.]+).*",
-    "F1": r"^{\"exact_match\"\:\s*[\d\.]+,\s*\"f1\"\:\s*([\d\.]+)}",
-    "WER": r"Word Error Rate\:.*, accuracy=([0-9\.]+)%",
-    "DICE": r"Accuracy\:\s*mean\s*=\s*([\d\.]+).*",
+    "acc":
+        r"^accuracy=([\d\.]+).*",
+    "AUC":
+        r"^AUC=([\d\.]+).*",
+    "mAP":
+        r"^mAP=([\d\.]+).*",
+    "bleu":
+        r"^BLEU\:\s*([\d\.]+).*",
+    "F1":
+        r"^{[\"\']exact_match[\"\']\:\s*[\d\.]+,\s*[\"\']f1[\"\']\:\s*([\d\.]+)}",
+    "WER":
+        r"Word Error Rate\:.*, accuracy=([0-9\.]+)%",
+    "DICE":
+        r"Accuracy\:\s*mean\s*=\s*([\d\.]+).*",
 }
 
 SYSTEM_DESC_REQUIRED_FIELDS = [