test: A/B-ify boottime test

Run the boottime test as an A/B-test post-PR so that we note changes to
this key metrics other than "are we below 150ms?".

Also incorporates the changes from #4316 so that we can run the
short-running boottime and memory overhead tests on the same .metal, to
avoid spawning yet another yet of 15 .metals after every PR merge.

Signed-off-by: Patrick Roy <roypat@amazon.co.uk>
Co-authored-by: Jonathan Woollett-Light <jcawl@amazon.co.uk>

.buildkite/pipeline_perf.py

@@ -45,10 +45,9 @@
         "devtool_opts": "-c 1-10 -m 0",
     },
     "memory-overhead": {
-        "label": "💾 Memory Overhead",
-        "test_path": "integration_tests/performance/test_memory_overhead.py",
+        "label": "💾 Memory Overhead and 👢 Boottime",
+        "test_path": "'integration_tests/performance/test_memory_overhead.py integration_tests/performance/test_boottime.py::test_boottime'",
         "devtool_opts": "-c 1-10 -m 0",
-        "ab_opts": "--noise-threshold 0.01",
     },
 }
 

tests/integration_tests/performance/test_boottime.py

@@ -173,6 +173,7 @@ def test_boottime(
     metrics.set_dimensions(
         {
             **DIMENSIONS,
+            "performance_test": "test_boottime",
             "guest_kernel": guest_kernel.name,
             "vcpus": str(vcpu_count),
             "mem_size_mib": str(mem_size_mib),