From fec79f048ad76c2db7b17a9ac9afe507d3930f95 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Thu, 11 Jan 2024 17:09:19 -0500 Subject: [PATCH 1/6] Post GitHub issue link only when the bisection is successful. --- .github/workflows/v3-bisection.yml | 9 ++ .github/workflows/v3-nightly.yml | 18 ++-- bisection.py | 132 ++++++++++++++--------------- regression_detector.py | 38 +++++++-- utils/github.py | 61 +++++++++++++ 5 files changed, 170 insertions(+), 88 deletions(-) create mode 100644 utils/github.py diff --git a/.github/workflows/v3-bisection.yml b/.github/workflows/v3-bisection.yml index ab35ee6765..53a5c495b8 100644 --- a/.github/workflows/v3-bisection.yml +++ b/.github/workflows/v3-bisection.yml @@ -81,6 +81,15 @@ jobs: --torchbench-repo-path "${PWD}" --config "${BISECT_WORKDIR}/regression-${REGRESSION_DATE}.yaml" \ --output "${BISECT_WORKDIR}/bisect-output-gh${GITHUB_RUN_ID}.json" cp -r "${BISECT_WORKDIR}" ../bisection-result + - name: Create the github issue + continue-on-error: true + if: env.TORCHBENCH_BISECTION_COMMIT_FOUND + uses: peter-evans/create-issue-from-file@v4 + with: + title: V3 Performance Signal Detected by TorchBench Userbenchmark "torch-nightly" on ${{ env.TORCHBENCH_BISECTION_COMMIT_FOUND }} + content-filepath: ./benchmark/gh-issue.md + labels: | + torchbench-perf-report - name: Upload artifact if: always() uses: actions/upload-artifact@v3 diff --git a/.github/workflows/v3-nightly.yml b/.github/workflows/v3-nightly.yml index f188a5b164..d47da0b239 100644 --- a/.github/workflows/v3-nightly.yml +++ b/.github/workflows/v3-nightly.yml @@ -70,15 +70,7 @@ jobs: done rm -r ../benchmark-output || true cp -r ./.userbenchmark/torch-nightly ../benchmark-output - - name: Create the github issue - continue-on-error: true - if: env.TORCHBENCH_REGRESSION_DETECTED - uses: peter-evans/create-issue-from-file@v4 - with: - title: V3 Performance Signal Detected by TorchBench Userbenchmark "torch-nightly" on ${{ env.TORCHBENCH_REGRESSION_DETECTED }} - content-filepath: ./benchmark/gh-issue.md - labels: | - torchbench-perf-report + - name: Copy artifact and upload to scribe and Amazon S3 run: | . "${SETUP_SCRIPT}" @@ -97,6 +89,14 @@ jobs: LATEST_REGRESSION_RESULT=$(find ../benchmark-output/ -name "regression-*.yaml" | sort -r | head -1) # Upload the regression json to Amazon S3 python ./scripts/userbenchmark/upload_s3.py --upload-file "${LATEST_REGRESSION_RESULT}" --userbenchmark_platform "${PLATFORM_NAME}" + # Get the workflow ID from + # https://api.github.com/repos/pytorch/benchmark/actions/workflows + # And dispatch the bisection workflow + curl -u xuzhao9:${{ secrets.TORCHBENCH_ACCESS_TOKEN }} \ + -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/pytorch/benchmark/actions/workflows/57994037/dispatches \ + -d '{"ref": "main", "inputs": {"regression_date": "${{ env.TORCHBENCH_REGRESSION_DETECTED }}" } }' - name: Upload result to GH Actions Artifact uses: actions/upload-artifact@v3 with: diff --git a/bisection.py b/bisection.py index 884bdeed19..67c9764eb9 100644 --- a/bisection.py +++ b/bisection.py @@ -21,12 +21,21 @@ from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple - import yaml from userbenchmark.utils import ( parse_abtest_result_from_regression_file_for_bisect, TorchBenchABTestResult, + parse_abtest_result_from_regression_file_for_bisect +) +from regression_detector import generate_regression_result +from utils import gitutils +from utils.github import process_bisection_into_gh_issue +from utils.build_utils import ( + setup_bisection_build_env, + build_repo, + cleanup_torch_packages, + TorchRepo, ) TORCHBENCH_BISECTION_TARGETS = { @@ -173,21 +182,26 @@ def __str__(self): class BisectionTargetRepo: repo: TorchRepo + # Start and end git hash start: str end: str + # Start and end version + start_version: str + end_version: str non_target_repos: List[TorchRepo] # generated in prep() bisection_env: os._Environ commits: List[Commit] # Map from commit SHA to its index in commits commit_dict: Dict[str, int] - - def __init__( - self, repo: TorchRepo, start: str, end: str, non_target_repos: List[TorchRepo] - ): + def __init__(self, repo: TorchRepo, start: str, end: str, + start_version: str, end_version: str, + non_target_repos: List[TorchRepo]): self.repo = repo self.start = start self.end = end + self.start_version = start_version + self.end_version = end_version self.non_target_repos = non_target_repos self.commits = [] self.commit_dict = dict() @@ -488,7 +502,9 @@ def output(self): json_obj = dict() json_obj["target_repo"] = self.target_repo.repo.name json_obj["start"] = self.target_repo.start + json_obj["start_version"] = self.target_repo.start_version json_obj["end"] = self.target_repo.end + json_obj["end_version"] = self.target_repo.end_version json_obj["result"] = [] for res in self.result: r = dict() @@ -501,49 +517,35 @@ def output(self): json_obj["result"].append(r) with open(self.output_json, "w") as outfile: json.dump(json_obj, outfile, indent=2) - print(f"Bisection successful. Result saved to {self.output_json}:") + print(f"Bisection successful. Result saved to {self.output_json}.") print(json_obj) def main() -> None: global SKIP_INSTALL_TORCHBENCH parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--work-dir", - required=True, - help="bisection working directory for logs and results", - type=exist_dir_path, - ) - parser.add_argument( - "--torch-repos-path", - required=True, - help="the directory of pytorch/* source code repositories, or fbcode repo if running internally", - type=exist_dir_path, - ) - parser.add_argument( - "--torchbench-repo-path", - default=None, - help="the directory of torchbench source code git repository, if None, use `args.torch_repo_path/benchmark`.", - type=exist_dir_path, - ) - parser.add_argument( - "--config", - required=True, - help="the regression dict output of regression_detector.py in YAML", - type=exist_file_path, - ) - parser.add_argument( - "--skip-install-torchbench", - action="store_true", - help="Skip installing torchbench", - ) - parser.add_argument("--output", required=True, help="the output json file") - parser.add_argument( - "--skip-update", - type=str, - default="torchbench", - help="Repositories to skip update.", - ) + parser.add_argument("--work-dir", + required=True, + help="bisection working directory for logs and results", + type=exist_dir_path) + parser.add_argument("--torch-repos-path", + required=True, + help="the directory of pytorch/* source code repositories", + type=exist_dir_path) + parser.add_argument("--torchbench-repo-path", + default=None, + help="the directory of torchbench source code git repository, if None, use `args.torch_repo_path/benchmark`.", + type=exist_dir_path) + parser.add_argument("--config", + required=True, + help="the regression dict output of regression_detector.py in YAML", + type=exist_file_path) + parser.add_argument("--skip-install-torchbench", action="store_true", help="Skip installing torchbench") + parser.add_argument("--output", + required=True, + help="the output json file") + parser.add_argument("--skip-update", type=str, default="torchbench", help="Repositories to skip update.") + parser.add_argument("--gh-issue-path", default="gh-issue.md", help="Output path to print the issue body") # by default, debug mode is disabled parser.add_argument( "--debug", @@ -587,33 +589,21 @@ def main() -> None: args.torch_repos_path, args.torchbench_repo_path, skip_update_repos ) target_repo = torch_repos[bisect_config.bisection] - start_hash = ( - gitutils.get_torch_main_commit( - target_repo.src_path.absolute(), - bisect_config.control_env["git_commit_hash"], - ) - if not IS_FBCODE - else bisect_config.control_env["git_commit_hash"] - ) - end_hash = ( - gitutils.get_torch_main_commit( - target_repo.src_path.absolute(), - bisect_config.treatment_env["git_commit_hash"], - ) - if not IS_FBCODE - else bisect_config.treatment_env["git_commit_hash"] - ) - - bisection = TorchBenchBisection( - workdir=args.work_dir, - torch_repos=torch_repos, - target_repo=target_repo, - start=start_hash, - end=end_hash, - bisect_config=bisect_config, - output_json=args.output, - debug=args.debug, - ) + start_hash = gitutils.get_torch_main_commit(target_repo.src_path.absolute(), bisect_config.control_env["git_commit_hash"]) + end_hash = gitutils.get_torch_main_commit(target_repo.src_path.absolute(), bisect_config.treatment_env["git_commit_hash"]) + + bisection = TorchBenchBisection(workdir=args.work_dir, + torch_repos=torch_repos, + target_repo=torch_repos[bisect_config.bisection], + start=start_hash, + end=end_hash, + start_version=bisect_config.control_env["pytorch_version"] + if "pytorch_version" in bisect_config.control_env else "N/A", + end_version=bisect_config.treatment_env["pytorch_version"] + if "pytorch_version" in bisect_config.treatment_env else "N/A", + bisect_config=bisect_config, + output_json=args.output, + debug=args.debug) if start_hash == end_hash: print(f"Start and end hash are the same: {start_hash}. Skip bisection") bisection.output() @@ -625,7 +615,9 @@ def main() -> None: ) bisection.run() bisection.output() - + # Format the output into a github issue if the bisector finds the root cause commit + if bisection.result: + process_bisection_into_gh_issue(bisection.output_json, args.gh_issue_path) if __name__ == "__main__": main() # pragma: no cover diff --git a/regression_detector.py b/regression_detector.py index bb28f39145..f2b58ca366 100644 --- a/regression_detector.py +++ b/regression_detector.py @@ -6,11 +6,12 @@ import importlib from dataclasses import asdict import os +import re import yaml from pathlib import Path import time from datetime import datetime -from typing import Any, List, Dict, Optional +from typing import Any, List, Dict, Tuple, Optional from userbenchmark.utils import PLATFORMS, USERBENCHMARK_OUTPUT_PREFIX, REPO_PATH, \ TorchBenchABTestResult, get_date_from_metrics, \ get_ub_name, get_latest_files_in_s3_from_last_n_days, get_date_from_metrics_s3_key @@ -19,17 +20,19 @@ GITHUB_ISSUE_TEMPLATE = """ TorchBench CI has detected a performance signal or runtime regression. -Base PyTorch commit: {start} +Control PyTorch commit: {control_commit} +Control PyTorch version: {control_version} -Affected PyTorch commit: {end} +Treatment PyTorch commit: {treatment_commit} +Treatment PyTorch version: {treatment_version} Affected Tests: {test_details} -Tests that were no longer run on affected commit: +Tests that were no longer run on treatment commit: {control_only_tests} -Tests that were newly added on affected commit: +Tests that were newly added on treatment commit: {treatment_only_tests} Runtime regressions found? @@ -103,6 +106,15 @@ def process_regressions_into_yaml(regression_result: TorchBenchABTestResult, out def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult, owner: str, output_path: str, errors_path: str) -> None: + def _parse_date_from_pytorch_version(pytorch_version: str) -> Optional[str]: + # example pytorch nightly version: "2.2.0.dev20231116+cu118" + # return a date string like "2023-11-16" + ver_regex = "dev[0-9+]\+" + s = re.search(ver_regex, pytorch_version) + if not s or not s.groups(): + return None + return datetime.strftime(datetime.strptime(s.groups[0], "%Y%m%d"), "%Y-%m-%d") + regressions_dict = asdict(regression_result) troubled_tests = "" for test, stats in regressions_dict["details"].items(): @@ -122,7 +134,9 @@ def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult, treatment_only_tests += f"- {test}: {stat}\n" control_commit = regressions_dict["control_env"]["pytorch_git_version"] + control_version = regressions_dict["control_env"]["pytorch_version"] treatment_commit = regressions_dict["treatment_env"]["pytorch_git_version"] + treatment_version = regressions_dict["treatment_env"]["pytorch_version"] runtime_regressions_msg = "No runtime errors were found in the " + \ "new benchmarks run--you are all good there!" @@ -138,7 +152,11 @@ def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult, if "GITHUB_ENV" in os.environ: fname = os.environ["GITHUB_ENV"] - content = f"TORCHBENCH_REGRESSION_DETECTED='{treatment_commit}'\n" + treatment_date = _parse_date_from_pytorch_version(treatment_version) + # If can't parse the version date from pytorch version, use today + if not treatment_date: + treatment_date = datetime.today().strftime("%Y-%m-%d") + content = f"TORCHBENCH_REGRESSION_DETECTED='{treatment_date}'\n" with open(fname, 'a') as fo: fo.write(content) @@ -149,8 +167,10 @@ def process_regressions_into_gh_issue(regression_result: TorchBenchABTestResult, github_run_url = f"https://github.com/pytorch/benchmark/actions/runs/{github_run_id}" issue_config: Dict[str, str] = { - "start": control_commit, - "end": treatment_commit, + "control_commit": control_commit, + "treatment_commit": treatment_commit, + "control_version": control_version, + "treatment_version": treatment_version, "test_details": troubled_tests, "control_only_tests": control_only_tests, "treatment_only_tests": treatment_only_tests, @@ -174,7 +194,7 @@ def get_best_start_date(latest_metrics_jsons: List[str], end_date: datetime) -> return None -def get_metrics_by_date(latest_metrics_jsons: List[str], pick_date: datetime): +def get_metrics_by_date(latest_metrics_jsons: List[str], pick_date: datetime) -> Tuple[Any, str]: pick_metrics_json_key: Optional[str] = None for metrics_json_key in latest_metrics_jsons: metric_datetime = get_date_from_metrics_s3_key(metrics_json_key) diff --git a/utils/github.py b/utils/github.py new file mode 100644 index 0000000000..47c5374033 --- /dev/null +++ b/utils/github.py @@ -0,0 +1,61 @@ +import json +import os + +from typing import Dict + +GITHUB_ISSUE_TEMPLATE = """ +TorchBench CI has detected a performance signal or runtime regression, and bisected its result. + +Control PyTorch commit: {control_commit} +Control PyTorch version: {control_version} + +Treatment PyTorch commit: {treatment_commit} +Treatment PyTorch version: {treatment_version} + +Bisection result: + +``` +{result} +``` + +cc {owner} +""" + +DEFAULT_GH_ISSUE_OWNER = "@xuzhao9" + +def process_bisection_into_gh_issue(bisection_output_json: str, output_path: str) -> None: + with open(bisection_output_json, "r") as fp: + bisection = json.load(fp) + + result = json.dump(bisection, indent=4) + control_commit = bisection["start"] + control_version = bisection["start_version"] + treatment_commit = bisection["end"] + treatment_version = bisection["end_version"] + + if "GITHUB_ENV" in os.environ: + fname = os.environ["GITHUB_ENV"] + content = f"TORCHBENCH_BISECTION_COMMIT_FOUND_OR_FAILED='{bisection.target_repo.end}'\n" + with open(fname, 'a') as fo: + fo.write(content) + process_bisection_into_gh_issue(bisection.output_json) + + github_run_id = os.environ.get("GITHUB_RUN_ID", None) + github_run_url = "No URL found, please look for the failing action in " + \ + "https://github.com/pytorch/benchmark/actions" + if github_run_id is not None: + github_run_url = f"https://github.com/pytorch/benchmark/actions/runs/{github_run_id}" + + issue_config: Dict[str, str] = { + "control_commit": control_commit, + "treatment_commit": treatment_commit, + "control_version": control_version, + "treatment_version": treatment_version, + "result": result, + "github_run_url": github_run_url, + "owner": DEFAULT_GH_ISSUE_OWNER + } + + issue_body = GITHUB_ISSUE_TEMPLATE.format(**issue_config) + with open(output_path, "w") as f: + f.write(issue_body) From 588f71a1254fa75f05de5f8589d37b062f5d70f0 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 20 Nov 2023 18:56:56 -0500 Subject: [PATCH 2/6] Add kickoff bisection --- .github/workflows/v3-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/v3-nightly.yml b/.github/workflows/v3-nightly.yml index d47da0b239..b3aefbdeab 100644 --- a/.github/workflows/v3-nightly.yml +++ b/.github/workflows/v3-nightly.yml @@ -81,7 +81,7 @@ jobs: python ./scripts/userbenchmark/upload_scribe.py --userbenchmark_json "${LATEST_RESULT}" --userbenchmark_platform "${PLATFORM_NAME}" # Upload the result json to Amazon S3 python ./scripts/userbenchmark/upload_s3.py --upload-file "${LATEST_RESULT}" --userbenchmark_platform "${PLATFORM_NAME}" - - name: Copy regression results to Amazon S3 + - name: Copy regression results to Amazon S3 and kick off bisection if: env.TORCHBENCH_REGRESSION_DETECTED run: | . "${SETUP_SCRIPT}" From ed854195afda2b94d9c1cdff831690948570d53c Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 20 Nov 2023 18:59:12 -0500 Subject: [PATCH 3/6] Remove empty line --- .github/workflows/v3-nightly.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/v3-nightly.yml b/.github/workflows/v3-nightly.yml index b3aefbdeab..670013b69d 100644 --- a/.github/workflows/v3-nightly.yml +++ b/.github/workflows/v3-nightly.yml @@ -70,7 +70,6 @@ jobs: done rm -r ../benchmark-output || true cp -r ./.userbenchmark/torch-nightly ../benchmark-output - - name: Copy artifact and upload to scribe and Amazon S3 run: | . "${SETUP_SCRIPT}" From 468e9ef5a0fc63d3857b38866bc2c979740d8f18 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Thu, 11 Jan 2024 17:13:16 -0500 Subject: [PATCH 4/6] Fix formatting issues. --- bisection.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/bisection.py b/bisection.py index 67c9764eb9..52c953960b 100644 --- a/bisection.py +++ b/bisection.py @@ -21,21 +21,12 @@ from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple + import yaml from userbenchmark.utils import ( parse_abtest_result_from_regression_file_for_bisect, TorchBenchABTestResult, - parse_abtest_result_from_regression_file_for_bisect -) -from regression_detector import generate_regression_result -from utils import gitutils -from utils.github import process_bisection_into_gh_issue -from utils.build_utils import ( - setup_bisection_build_env, - build_repo, - cleanup_torch_packages, - TorchRepo, ) TORCHBENCH_BISECTION_TARGETS = { @@ -72,7 +63,7 @@ TorchRepo, ) from utils.cuda_utils import DEFAULT_CUDA_VERSION, prepare_cuda_env - + from utils.github import process_bisection_into_gh_issue IS_FBCODE = False except (ImportError, ModuleNotFoundError): # Meta-Internal imports From aa5fa8e0f0534d5e60a062ec5ec776dea40d8674 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Thu, 11 Jan 2024 19:15:48 -0500 Subject: [PATCH 5/6] Fix bisection code --- bisection.py | 63 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/bisection.py b/bisection.py index 52c953960b..dc1f0453ba 100644 --- a/bisection.py +++ b/bisection.py @@ -515,28 +515,47 @@ def output(self): def main() -> None: global SKIP_INSTALL_TORCHBENCH parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--work-dir", - required=True, - help="bisection working directory for logs and results", - type=exist_dir_path) - parser.add_argument("--torch-repos-path", - required=True, - help="the directory of pytorch/* source code repositories", - type=exist_dir_path) - parser.add_argument("--torchbench-repo-path", - default=None, - help="the directory of torchbench source code git repository, if None, use `args.torch_repo_path/benchmark`.", - type=exist_dir_path) - parser.add_argument("--config", - required=True, - help="the regression dict output of regression_detector.py in YAML", - type=exist_file_path) - parser.add_argument("--skip-install-torchbench", action="store_true", help="Skip installing torchbench") - parser.add_argument("--output", - required=True, - help="the output json file") - parser.add_argument("--skip-update", type=str, default="torchbench", help="Repositories to skip update.") - parser.add_argument("--gh-issue-path", default="gh-issue.md", help="Output path to print the issue body") + parser.add_argument( + "--work-dir", + required=True, + help="bisection working directory for logs and results", + type=exist_dir_path, + ) + parser.add_argument( + "--torch-repos-path", + required=True, + help="the directory of pytorch/* source code repositories, or fbcode repo if running internally", + type=exist_dir_path, + ) + parser.add_argument( + "--torchbench-repo-path", + default=None, + help="the directory of torchbench source code git repository, if None, use `args.torch_repo_path/benchmark`.", + type=exist_dir_path, + ) + parser.add_argument( + "--config", + required=True, + help="the regression dict output of regression_detector.py in YAML", + type=exist_file_path, + ) + parser.add_argument( + "--skip-install-torchbench", + action="store_true", + help="Skip installing torchbench", + ) + parser.add_argument("--output", required=True, help="the output json file") + parser.add_argument( + "--skip-update", + type=str, + default="torchbench", + help="Repositories to skip update.", + ) + parser.add_argument( + "--gh-issue-path", + default="gh-issue.md", + help="Output path to print the issue body" + ) # by default, debug mode is disabled parser.add_argument( "--debug", From 6b57f6b0ff70ca3dfc4a26e3a366e56157966258 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Fri, 12 Jan 2024 13:51:38 -0500 Subject: [PATCH 6/6] Fix the bisection --- bisection.py | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/bisection.py b/bisection.py index dc1f0453ba..0758347eb5 100644 --- a/bisection.py +++ b/bisection.py @@ -599,21 +599,35 @@ def main() -> None: args.torch_repos_path, args.torchbench_repo_path, skip_update_repos ) target_repo = torch_repos[bisect_config.bisection] - start_hash = gitutils.get_torch_main_commit(target_repo.src_path.absolute(), bisect_config.control_env["git_commit_hash"]) - end_hash = gitutils.get_torch_main_commit(target_repo.src_path.absolute(), bisect_config.treatment_env["git_commit_hash"]) - - bisection = TorchBenchBisection(workdir=args.work_dir, - torch_repos=torch_repos, - target_repo=torch_repos[bisect_config.bisection], - start=start_hash, - end=end_hash, - start_version=bisect_config.control_env["pytorch_version"] - if "pytorch_version" in bisect_config.control_env else "N/A", - end_version=bisect_config.treatment_env["pytorch_version"] - if "pytorch_version" in bisect_config.treatment_env else "N/A", - bisect_config=bisect_config, - output_json=args.output, - debug=args.debug) + start_hash = ( + gitutils.get_torch_main_commit( + target_repo.src_path.absolute(), + bisect_config.control_env["git_commit_hash"], + ) + if not IS_FBCODE + else bisect_config.control_env["git_commit_hash"] + ) + end_hash = ( + gitutils.get_torch_main_commit( + target_repo.src_path.absolute(), + bisect_config.treatment_env["git_commit_hash"], + ) + if not IS_FBCODE + else bisect_config.treatment_env["git_commit_hash"] + ) + + bisection = TorchBenchBisection( + workdir=args.work_dir, + torch_repos=torch_repos, + target_repo=target_repo, + start=start_hash, + end=end_hash, + start_version=bisect_config.control_env.get("pytorch_version", "N/A"), + end_version=bisect_config.treatment_env.get("pytorch_version", "N/A"), + bisect_config=bisect_config, + output_json=args.output, + debug=args.debug, + ) if start_hash == end_hash: print(f"Start and end hash are the same: {start_hash}. Skip bisection") bisection.output()