Benchmarking Workflows #85
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled, unlabeled] | |
| workflow_dispatch: | |
| jobs: | |
| libor: | |
| #if: contains(github.event.pull_request.labels.*.name, 'benchmarks') || github.event_name == 'workflow_dispatch' | |
| name: Libor Swaption Benchmarks | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/foonathan/gcc:12 | |
| steps: | |
| - name: Hardware characteristics | |
| run: | | |
| echo "----------------- CPU ----------------" | |
| lscpu | |
| echo "------------------ Mem ---------------" | |
| lsmem | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pr | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| ref: main | |
| - name: Setup | |
| run: | | |
| mkdir pr/build main/build | |
| echo "Setup complete" | |
| - name: Reference | |
| working-directory: main | |
| run: | | |
| set -e | |
| cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=Release | |
| cmake --build build | |
| cd build/samples/LiborSwaptionPricer | |
| ls -l | |
| file ./LiborSwaptionPricer | |
| chmod +x ./LiborSwaptionPricer | |
| rm -f reference.log || true | |
| # warmup run | |
| ./LiborSwaptionPricer 50000 | |
| for i in $(seq 1 ${{ vars.REPETITIONS }}) ; do \ | |
| ./LiborSwaptionPricer 50000 | tee -a reference.log ; \ | |
| done | |
| - name: Benchmark | |
| working-directory: pr | |
| run: | | |
| set -e | |
| cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=Release | |
| cmake --build build | |
| cd build/samples/LiborSwaptionPricer | |
| rm -f benchmark.log || true | |
| # warmup run | |
| ./LiborSwaptionPricer 50000 | |
| for i in $(seq 1 ${{ vars.REPETITIONS }}) ; do \ | |
| ./LiborSwaptionPricer 50000 | tee -a benchmark.log ; \ | |
| done | |
| - name: Compare | |
| id: compare | |
| run: | | |
| set -e | |
| apt-get update && apt-get install -y bc datamash --fix-missing | |
| BENCHMARK_LOG="pr/build/samples/LiborSwaptionPricer/benchmark.log" | |
| REFERENCE_LOG="main/build/samples/LiborSwaptionPricer/reference.log" | |
| # Extract results and medians | |
| OUTPUT_RESULTS=$(awk '$2 == "AAD" { print $4 }' "$BENCHMARK_LOG" | datamash min 1 max 1 mean 1 sstdev 1 median 1 trimmean 1 geomean 1 harmmean 1) | |
| OUT_TIME=$(awk '$2 == "AAD" { print $4 }' "$BENCHMARK_LOG" | datamash median 1 | awk '{printf "%.2f", $1}') | |
| REFERENCE_RESULTS=$(awk '$2 == "AAD" { print $4 }' "$REFERENCE_LOG" | datamash min 1 max 1 mean 1 sstdev 1 median 1 trimmean 1 geomean 1 harmmean 1) | |
| REF_TIME=$(awk '$2 == "AAD" { print $4 }' "$REFERENCE_LOG" | datamash median 1 | awk '{printf "%.2f", $1}') | |
| # Compute differences and percentage changes | |
| DIFF=$(echo "$REF_TIME - $OUT_TIME" | bc | awk '{printf "%.2f", $1}') | |
| PERCENTAGE_CHANGE=$(awk "BEGIN { printf \"%.2f\", (($DIFF / $OUT_TIME) * 100) }") | |
| # Generate the Markdown table | |
| cat <<EOF > benchmark_results.md | |
| # Libor Benchmarks | |
| | Metric | Runs | Reference (us) | Benchmark (us) | Difference (us) | % Change | | |
| |:---------- | ---------:| --------------:| --------------:| ---------------:| --------:| | |
| | Median | ${{ vars.REPETITIONS }} | $REF_TIME | $OUT_TIME | $DIFF | $PERCENTAGE_CHANGE% | | |
| EOF | |
| echo "RESULTS_FILE=$(pwd)/benchmark_results.md" >> $GITHUB_ENV | |
| - name: Comment on PR | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-path: ${{ env.RESULTS_FILE }} | |
| env: | |
| RESULTS_FILE: ${{ env.RESULTS_FILE }} | |
| quantlib: | |
| #if: contains(github.event.pull_request.labels.*.name, 'benchmarks') || github.event_name == 'workflow_dispatch' | |
| name: QuantLib Benchmarks | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/lballabio/quantlib-devenv:noble-1.85.0 | |
| env: | |
| TESTS: "testPathwiseGreeks testSwaptionVolMatrixCoherence AdjointBermudanSwaption" | |
| steps: | |
| - name: Hardware characteristics | |
| run: | | |
| echo "----------------- CPU ----------------" | |
| lscpu | |
| echo "------------------ Mem ---------------" | |
| lsmem | |
| - name: Checkout XAD Main | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pr | |
| - name: Checkout QuantLib | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: lballabio/QuantLib | |
| ref: v1.35 | |
| path: QuantLib | |
| - name: Checkout Risks-Cpp | |
| uses: actions/checkout@v4 | |
| with: | |
| path: QuantLib-Risks-Cpp | |
| repository: auto-differentiation/QuantLib-Risks-Cpp | |
| ref: a1fcf4f0fab0a464710c249aa76871421675babd | |
| - name: ccache | |
| uses: hendrikmuhs/ccache-action@v1.2 | |
| with: | |
| key: benchmark-quantlib | |
| - name: Setup | |
| run: | | |
| apt-get update \ | |
| && apt-get install -y ninja-build bc datamash unzip | |
| - name: Reference Build | |
| run: | | |
| set -e | |
| cd QuantLib | |
| mkdir -p build | |
| cd build | |
| cmake -G Ninja -DBOOST_ROOT=/usr \ | |
| -DCMAKE_CXX_STANDARD=17 \ | |
| -DQLRISKS_DISABLE_AAD=OFF \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DQL_EXTERNAL_SUBDIRECTORIES="$(pwd)/../../main;$(pwd)/../../QuantLib-Risks-Cpp" \ | |
| -DQL_EXTRA_LINK_LIBRARIES=QuantLib-Risks \ | |
| -DQL_NULL_AS_FUNCTIONS=ON \ | |
| -DXAD_NO_THREADLOCAL=ON \ | |
| -DXAD_SIMD_OPTION=AVX2 \ | |
| .. | |
| cmake --build . | |
| - name: Reference Runs | |
| run: | | |
| chmod +x ./main/scripts/run_benchmark.sh | |
| bash ./main/scripts/run_benchmark.sh reference ${{ vars.REPETITIONS }} ${{ env.TESTS }} | |
| - name: Benchmark Build | |
| run: | | |
| set -e | |
| cd QuantLib | |
| mkdir -p benchmark-build | |
| cd benchmark-build | |
| cmake -G Ninja -DBOOST_ROOT=/usr \ | |
| -DCMAKE_CXX_STANDARD=17 \ | |
| -DQLRISKS_DISABLE_AAD=OFF \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DQL_EXTERNAL_SUBDIRECTORIES="$(pwd)/../../pr;$(pwd)/../../QuantLib-Risks-Cpp" \ | |
| -DQL_EXTRA_LINK_LIBRARIES=QuantLib-Risks \ | |
| -DQL_NULL_AS_FUNCTIONS=ON \ | |
| -DXAD_NO_THREADLOCAL=ON \ | |
| -DXAD_SIMD_OPTION=AVX2 \ | |
| .. | |
| cmake --build . | |
| - name: Benchmark Runs | |
| run: | | |
| chmod +x ./main/scripts/run_benchmark.sh | |
| bash ./main/scripts/run_benchmark.sh benchmark ${{ vars.REPETITIONS }} ${{ env.TESTS }} | |
| - name: Compare | |
| id: compare | |
| run: | | |
| apt-get update && apt-get install -y bc datamash --fix-missing | |
| chmod +x ./main/scripts/compare_benchmark.sh | |
| bash ./main/scripts/compare_benchmark.sh ${{ env.TESTS }} | |
| - name: Comment on PR | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-path: ${{ env.RESULTS_FILE }} | |
| env: | |
| RESULTS_FILE: ${{ env.RESULTS_FILE }} |