Skip to content

Commit

Permalink
Fix another bug in AIBench local bisector
Browse files Browse the repository at this point in the history
Summary:
Fix the regression detector by importing the correct package name.
Deal with the case when the base commit metric could be zero.

Reviewed By: aaronenyeshi

Differential Revision: D51866054

fbshipit-source-id: 811dbdc688ce428bc000cebcf9402e651ca840b6
  • Loading branch information
xuzhao9 authored and facebook-github-bot committed Dec 6, 2023
1 parent f660f4f commit a13c590
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 7 deletions.
5 changes: 3 additions & 2 deletions bisection.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,9 @@ def get_digest_for_commit(self, commit: Commit, abtest_result: Dict[str, Any], d
# Run benchmark, return the output json file
result_json = self._run_benchmark_for_commit(commit, abtest_result)
commit.digest = self._gen_digest(result_json)
print(f"================== [TORCHBENCH] Cleaning up packages for commit {commit.sha} ==================", flush=True)
cleanup_torch_packages()
if not IS_FBCODE:
print(f"================== [TORCHBENCH] Cleaning up packages for commit {commit.sha} ==================", flush=True)
cleanup_torch_packages()
return commit.digest

class TorchBenchBisection:
Expand Down
13 changes: 8 additions & 5 deletions regression_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,11 @@ def _call_userbenchmark_detector(detector, control: Dict[str, Any], treatment: D
assert control["name"] == treatment["name"], f'Expected the same userbenchmark name from metrics files, \
but getting {control["name"]} and {treatment["name"]}.'
bm_name = control["name"]
detector = importlib.import_module(f"userbenchmark.{bm_name}.regression_detector").run

try:
detector = importlib.import_module(f"userbenchmark.{bm_name}.regression_detector").run
except:
# fbcode
detector = importlib.import_module(f"userbenchmark.fb.{bm_name}.regression_detector").run
# Process control and treatment to include only shared keys
filtered_control_metrics = {}
control_only_metrics = {}
Expand Down Expand Up @@ -97,7 +100,7 @@ def process_regressions_into_yaml(regression_result: TorchBenchABTestResult, out
with open(output_path, "w") as ofptr:
ofptr.write(output_yaml_str)
print(f"Wrote above yaml to {output_path}.")


def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult, owner: str, output_path: str, errors_path: str) -> None:
regressions_dict = asdict(regression_result)
Expand All @@ -109,7 +112,7 @@ def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult,
troubled_tests += f"- {test}: {sign}{delta:.5%}\n"
else:
troubled_tests += f"- {test}: {delta}\n"

control_only_tests = ""
for test, stat in regressions_dict["control_only_metrics"].items():
control_only_tests += f"- {test}: {stat}\n"
Expand Down Expand Up @@ -155,7 +158,7 @@ def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult,
"github_run_url": github_run_url,
"owner": owner
}

issue_body = GITHUB_ISSUE_TEMPLATE.format(**issue_config)
print(issue_body)
with open(output_path, "w") as f:
Expand Down

0 comments on commit a13c590

Please sign in to comment.