diff --git a/.buildbot.sh b/.buildbot.sh
index eeedea7c8..f20051184 100644
--- a/.buildbot.sh
+++ b/.buildbot.sh
@@ -210,9 +210,20 @@ for tracer in $TRACERS; do
     echo "===> Running ${tracer} tests"
     RUST_TEST_SHUFFLE=1 cargo test --release
 
-    # test yklua/hwt in release mode.
     if [ "${tracer}" = "hwt" ]; then
+        # test yklua/hwt in release mode.
         PATH=$(pwd)/bin:${PATH} YK_BUILD_TYPE=release YKB_TRACER=hwt test_yklua
+
+        # Do a quick run of the benchmark suite as a smoke test.
+        pipx install rebench
+        git clone https://github.com/ykjit/yk-benchmarks
+        cd yk-benchmarks
+        ln -s ../yklua .
+        sed -e 's/executions: \[Lua, YkLua\]/executions: [YkLua]/' \
+            -e 's/executable: yklua/executable: lua/' \
+            rebench.conf > rebench2.conf
+        ~/.local/bin/rebench --quick --no-denoise -c rebench2.conf
+        cd ..
     fi
 done
 
diff --git a/.buildbot_dockerfile_debian b/.buildbot_dockerfile_debian
index 414550979..480816120 100644
--- a/.buildbot_dockerfile_debian
+++ b/.buildbot_dockerfile_debian
@@ -5,7 +5,7 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
   rm -f /etc/apt/apt.conf.d/docker-clean && \
   apt-get update && \
   apt-get -y install clang-15 make curl procps file git cmake python3 \
-    libtinfo-dev libzip-dev mold ninja-build gdb && \
+    libtinfo-dev libzip-dev mold ninja-build gdb pipx && \
   update-alternatives --install /usr/bin/cc cc /usr/bin/clang-15 999 && \
   update-alternatives --set cc /usr/bin/clang-15 && \
   update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-15 999 && \