From ca18f5c39e65ae2ef3e05b293aa6f960fb6340e1 Mon Sep 17 00:00:00 2001 From: Tom Jablin Date: Tue, 22 Mar 2022 10:33:56 -0700 Subject: [PATCH] Add an accuracy column for open submissions --- tools/submission/generate-final-report.py | 1 + tools/submission/submission-checker.py | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/submission/generate-final-report.py b/tools/submission/generate-final-report.py index 39535f8ac..beb7c4e7a 100644 --- a/tools/submission/generate-final-report.py +++ b/tools/submission/generate-final-report.py @@ -81,6 +81,7 @@ def main(): indices['open'] = indices['closed'].copy() indices['closed'].append('Details') indices['open'].append('UsedModel') + indices['open'].append('Accuracy') indices['open'].append('Details') columns = [ 'Model', diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py index 6b71e029d..ba1a262e2 100755 --- a/tools/submission/submission-checker.py +++ b/tools/submission/submission-checker.py @@ -612,13 +612,20 @@ } ACC_PATTERN = { - "acc": r"^accuracy=([\d\.]+).*", - "AUC": r"^AUC=([\d\.]+).*", - "mAP": r"^mAP=([\d\.]+).*", - "bleu": r"^BLEU\:\s*([\d\.]+).*", - "F1": r"^{\"exact_match\"\:\s*[\d\.]+,\s*\"f1\"\:\s*([\d\.]+)}", - "WER": r"Word Error Rate\:.*, accuracy=([0-9\.]+)%", - "DICE": r"Accuracy\:\s*mean\s*=\s*([\d\.]+).*", + "acc": + r"^accuracy=([\d\.]+).*", + "AUC": + r"^AUC=([\d\.]+).*", + "mAP": + r"^mAP=([\d\.]+).*", + "bleu": + r"^BLEU\:\s*([\d\.]+).*", + "F1": + r"^{[\"\']exact_match[\"\']\:\s*[\d\.]+,\s*[\"\']f1[\"\']\:\s*([\d\.]+)}", + "WER": + r"Word Error Rate\:.*, accuracy=([0-9\.]+)%", + "DICE": + r"Accuracy\:\s*mean\s*=\s*([\d\.]+).*", } SYSTEM_DESC_REQUIRED_FIELDS = [