diff --git a/.github/workflows/userbenchmark-regression-detector.yml b/.github/workflows/userbenchmark-regression-detector.yml index 3d7261e425..fe5146c4ff 100644 --- a/.github/workflows/userbenchmark-regression-detector.yml +++ b/.github/workflows/userbenchmark-regression-detector.yml @@ -50,7 +50,7 @@ jobs: if [ -d .userbenchmark ]; then rm -Rf .userbenchmark; fi # TODO: scale this to run other benchmarks, but let's start with optim - python -m userbenchmark.optim.run -m BERT_pytorch -d cuda -o AdamW --df no_foreach -f pt2_ + python -m userbenchmark.optim.run_optim_benchmarks -c ${{ github.event.inputs.userbenchmark_options }} cp -r ./.userbenchmark/optim ../benchmark-output - name: Detect potential regressions continue-on-error: true diff --git a/userbenchmark/optim/run_optim_benchmarks.py b/userbenchmark/optim/run_optim_benchmarks.py index a264ef2c26..097d7d30a6 100644 --- a/userbenchmark/optim/run_optim_benchmarks.py +++ b/userbenchmark/optim/run_optim_benchmarks.py @@ -53,6 +53,14 @@ def main() -> None: assert not OUTPUT_DIR.exists() or not any(OUTPUT_DIR.glob("*")), \ f'{OUTPUT_DIR} must be empty or nonexistent. Its contents will be wiped by this script.' + command = [sys.executable, '-m', 'userbenchmark.optim.run', '-m', 'BERT_pytorch', '-d', 'cuda', '-o', 'Adam', '--df', 'no_foreach', '-f', 'pt2_'] + completed_process = subprocess.run(command, check=True) + # While it is certainly unexpected for a subprocess to fail, we don't want to halt entirely + # as there can be valuable benchmarks to gather from the other subprocesses. + if completed_process.returncode != 0: + print(f'OH NO, the subprocess for model {m} and device {d} exited with {completed_process.returncode}!') + + return # Run benchmarks in subprocesses to take isolate contexts and memory for m, d in itertools.product(args.models, args.devices): command = [sys.executable, '-m', 'userbenchmark.optim.run', '--continue-on-error',