Bisect new nightly error and fix the broken T4 workflow. (#2153)

Summary:
On 2024-02-06, the nightly workflow failed with error: https://github.com/pytorch/benchmark/actions/runs/7802969964
Improve the A100 bisection workflow to auto-bisect this error.

Pull Request resolved: #2153

Test Plan:
1. Bisection workflow: https://github.com/pytorch/benchmark/actions/runs/7805867410
2. T4 userbenchmark workflow: https://github.com/pytorch/benchmark/actions/runs/7805761391

Reviewed By: aaronenyeshi

Differential Revision: D53517311

Pulled By: xuzhao9

fbshipit-source-id: 0e556927f0344e56f0039effb33db8839bc153e1

userbenchmark/test_bench/run.py

@@ -174,6 +174,12 @@ def run_config(
     except OSError as e:
         print(" [oserror]", flush=True)
         return dict.fromkeys(metrics, str(e))
+    except KeyboardInterrupt:
+        print(" [user_interrupt]")
+        exit(1)
+    except Exception as e:
+        print(" [runtime_error]", flush=True)
+        return dict.fromkeys(metrics, str(e))
 
 
 def run_config_accuracy(
@@ -196,7 +202,12 @@ def run_config_accuracy(
     except OSError as e:
         print(" [oserror]", flush=True)
         return {"accuracy": str(e)}
-
+    except KeyboardInterrupt:
+        print(" [user_interrupt]")
+        exit(1)
+    except Exception as e:
+        print(" [runtime_error]", flush=True)
+        return {"accuracy": str(e)}
 
 def parse_known_args(args):
     parser = argparse.ArgumentParser()

userbenchmark/utils.py

@@ -104,6 +104,7 @@ def get_output_dir(bm_name: str) -> Path:
 def get_default_output_json_path(bm_name: str, target_dir: Path=None) -> str:
     if target_dir is None:
         target_dir = get_output_dir(bm_name)
+    target_dir.mkdir(exist_ok=True, parents=True)
     fname = "metrics-{}.json".format(datetime.fromtimestamp(time.time()).strftime("%Y%m%d%H%M%S"))
     full_fname = os.path.join(target_dir, fname)
     return full_fname