From a13c590f51e9a5d0ee7357c2967934333dfdf654 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Wed, 6 Dec 2023 15:17:27 -0800 Subject: [PATCH] Fix another bug in AIBench local bisector Summary: Fix the regression detector by importing the correct package name. Deal with the case when the base commit metric could be zero. Reviewed By: aaronenyeshi Differential Revision: D51866054 fbshipit-source-id: 811dbdc688ce428bc000cebcf9402e651ca840b6 --- bisection.py | 5 +++-- regression_detector.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bisection.py b/bisection.py index fbaf7114ce..d5bc52b8aa 100644 --- a/bisection.py +++ b/bisection.py @@ -290,8 +290,9 @@ def get_digest_for_commit(self, commit: Commit, abtest_result: Dict[str, Any], d # Run benchmark, return the output json file result_json = self._run_benchmark_for_commit(commit, abtest_result) commit.digest = self._gen_digest(result_json) - print(f"================== [TORCHBENCH] Cleaning up packages for commit {commit.sha} ==================", flush=True) - cleanup_torch_packages() + if not IS_FBCODE: + print(f"================== [TORCHBENCH] Cleaning up packages for commit {commit.sha} ==================", flush=True) + cleanup_torch_packages() return commit.digest class TorchBenchBisection: diff --git a/regression_detector.py b/regression_detector.py index 7a0cbe535a..bb28f39145 100644 --- a/regression_detector.py +++ b/regression_detector.py @@ -55,8 +55,11 @@ def _call_userbenchmark_detector(detector, control: Dict[str, Any], treatment: D assert control["name"] == treatment["name"], f'Expected the same userbenchmark name from metrics files, \ but getting {control["name"]} and {treatment["name"]}.' bm_name = control["name"] - detector = importlib.import_module(f"userbenchmark.{bm_name}.regression_detector").run - + try: + detector = importlib.import_module(f"userbenchmark.{bm_name}.regression_detector").run + except: + # fbcode + detector = importlib.import_module(f"userbenchmark.fb.{bm_name}.regression_detector").run # Process control and treatment to include only shared keys filtered_control_metrics = {} control_only_metrics = {} @@ -97,7 +100,7 @@ def process_regressions_into_yaml(regression_result: TorchBenchABTestResult, out with open(output_path, "w") as ofptr: ofptr.write(output_yaml_str) print(f"Wrote above yaml to {output_path}.") - + def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult, owner: str, output_path: str, errors_path: str) -> None: regressions_dict = asdict(regression_result) @@ -109,7 +112,7 @@ def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult, troubled_tests += f"- {test}: {sign}{delta:.5%}\n" else: troubled_tests += f"- {test}: {delta}\n" - + control_only_tests = "" for test, stat in regressions_dict["control_only_metrics"].items(): control_only_tests += f"- {test}: {stat}\n" @@ -155,7 +158,7 @@ def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult, "github_run_url": github_run_url, "owner": owner } - + issue_body = GITHUB_ISSUE_TEMPLATE.format(**issue_config) print(issue_body) with open(output_path, "w") as f: