Benchmarking Workflows #89
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled, unlabeled] | |
| workflow_dispatch: | |
| jobs: | |
| libor: | |
| #if: contains(github.event.pull_request.labels.*.name, 'benchmarks') || github.event_name == 'workflow_dispatch' | |
| name: Libor Swaption Benchmarks | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/foonathan/gcc:12 | |
| steps: | |
| - name: Hardware characteristics | |
| run: | | |
| echo "----------------- CPU ----------------" | |
| lscpu | |
| echo "------------------ Mem ---------------" | |
| lsmem | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pr | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| ref: main | |
| - name: Setup | |
| run: | | |
| mkdir pr/build main/build | |
| echo "Setup complete" | |
| - name: Reference | |
| working-directory: main | |
| run: | | |
| set -e | |
| cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=Release | |
| cmake --build build | |
| cd build/samples/LiborSwaptionPricer | |
| ls -l | |
| file ./LiborSwaptionPricer | |
| chmod +x ./LiborSwaptionPricer | |
| rm -f reference.log || true | |
| # warmup run | |
| ./LiborSwaptionPricer 50000 | |
| for i in $(seq 1 ${{ vars.REPETITIONS || 3 }}) ; do \ | |
| ./LiborSwaptionPricer 50000 | tee -a reference.log ; \ | |
| done | |
| - name: Benchmark | |
| working-directory: pr | |
| run: | | |
| set -e | |
| cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=Release | |
| cmake --build build | |
| cd build/samples/LiborSwaptionPricer | |
| rm -f benchmark.log || true | |
| # warmup run | |
| ./LiborSwaptionPricer 50000 | |
| for i in $(seq 1 ${{ vars.REPETITIONS || 3 }}) ; do \ | |
| ./LiborSwaptionPricer 50000 | tee -a benchmark.log ; \ | |
| done | |
| - name: Compare | |
| id: compare | |
| run: | | |
| set -e | |
| apt-get update && apt-get install -y bc datamash --fix-missing | |
| BENCHMARK_LOG="pr/build/samples/LiborSwaptionPricer/benchmark.log" | |
| REFERENCE_LOG="main/build/samples/LiborSwaptionPricer/reference.log" | |
| OUTPUT_RESULTS=$(awk '$2 == "AAD" { print $4 }' "$BENCHMARK_LOG" | datamash min 1 max 1 mean 1 sstdev 1 median 1 trimmean 1 geomean 1 harmmean 1) | |
| OUT_TIME=$(awk '$2 == "AAD" { print $4 }' "$BENCHMARK_LOG" | datamash median 1 | awk '{printf "%.2f", $1}') | |
| REFERENCE_RESULTS=$(awk '$2 == "AAD" { print $4 }' "$REFERENCE_LOG" | datamash min 1 max 1 mean 1 sstdev 1 median 1 trimmean 1 geomean 1 harmmean 1) | |
| REF_TIME=$(awk '$2 == "AAD" { print $4 }' "$REFERENCE_LOG" | datamash median 1 | awk '{printf "%.2f", $1}') | |
| DIFF=$(echo "$REF_TIME - $OUT_TIME" | bc | awk '{printf "%.2f", $1}') | |
| PERCENTAGE_CHANGE=$(awk "BEGIN { printf \"%.2f\", (($DIFF / $OUT_TIME) * 100) }") | |
| cat <<EOF > libor_benchmark_results.md | |
| # Libor Benchmarks | |
| | Metric | Runs | Reference (us) | Benchmark (us) | Difference (us) | % Change | | |
| |:---------- | ---------:| --------------:| --------------:| ---------------:| --------:| | |
| | Median | ${{ vars.REPETITIONS || 3 }} | $REF_TIME | $OUT_TIME | $DIFF | $PERCENTAGE_CHANGE% | | |
| EOF | |
| echo "RESULTS_FILE=$(pwd)/libor_benchmark_results.md" >> $GITHUB_ENV | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: libor-benchmark-results | |
| path: libor_benchmark_results.md | |
| quantlib: | |
| #if: contains(github.event.pull_request.labels.*.name, 'benchmarks') || github.event_name == 'workflow_dispatch' | |
| name: QuantLib Benchmarks | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/lballabio/quantlib-devenv:noble-1.85.0 | |
| env: | |
| TESTS: "testPathwiseGreeks testSwaptionVolMatrixCoherence AdjointBermudanSwaption" | |
| steps: | |
| - name: Hardware characteristics | |
| run: | | |
| echo "----------------- CPU ----------------" | |
| lscpu | |
| echo "------------------ Mem ---------------" | |
| lsmem | |
| - name: Checkout XAD Main | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pr | |
| - name: Checkout QuantLib | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: lballabio/QuantLib | |
| ref: v1.35 | |
| path: QuantLib | |
| - name: Checkout Risks-Cpp | |
| uses: actions/checkout@v4 | |
| with: | |
| path: QuantLib-Risks-Cpp | |
| repository: auto-differentiation/QuantLib-Risks-Cpp | |
| ref: a1fcf4f0fab0a464710c249aa76871421675babd | |
| - name: ccache | |
| uses: hendrikmuhs/ccache-action@v1.2 | |
| with: | |
| key: benchmark-quantlib | |
| - name: Setup | |
| run: | | |
| apt-get update \ | |
| && apt-get install -y ninja-build bc datamash unzip | |
| - name: Reference Build | |
| run: | | |
| set -e | |
| cd QuantLib | |
| mkdir -p build | |
| cd build | |
| cmake -G Ninja -DBOOST_ROOT=/usr \ | |
| -DCMAKE_CXX_STANDARD=17 \ | |
| -DQLRISKS_DISABLE_AAD=OFF \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DQL_EXTERNAL_SUBDIRECTORIES="$(pwd)/../../main;$(pwd)/../../QuantLib-Risks-Cpp" \ | |
| -DQL_EXTRA_LINK_LIBRARIES=QuantLib-Risks \ | |
| -DQL_NULL_AS_FUNCTIONS=ON \ | |
| -DXAD_NO_THREADLOCAL=ON \ | |
| -DXAD_SIMD_OPTION=AVX2 \ | |
| .. | |
| cmake --build . | |
| - name: Reference Runs | |
| run: | | |
| chmod +x ./main/scripts/run_benchmark.sh | |
| bash ./main/scripts/run_benchmark.sh "reference" ${{ vars.REPETITIONS || 3 }} ${{ env.TESTS }} | |
| - name: Benchmark Build | |
| run: | | |
| set -e | |
| cd QuantLib | |
| mkdir -p benchmark-build | |
| cd benchmark-build | |
| cmake -G Ninja -DBOOST_ROOT=/usr \ | |
| -DCMAKE_CXX_STANDARD=17 \ | |
| -DQLRISKS_DISABLE_AAD=OFF \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DQL_EXTERNAL_SUBDIRECTORIES="$(pwd)/../../pr;$(pwd)/../../QuantLib-Risks-Cpp" \ | |
| -DQL_EXTRA_LINK_LIBRARIES=QuantLib-Risks \ | |
| -DQL_NULL_AS_FUNCTIONS=ON \ | |
| -DXAD_NO_THREADLOCAL=ON \ | |
| -DXAD_SIMD_OPTION=AVX2 \ | |
| .. | |
| cmake --build . | |
| - name: Benchmark Runs | |
| run: | | |
| chmod +x ./main/scripts/run_benchmark.sh | |
| bash ./main/scripts/run_benchmark.sh "benchmark" ${{ vars.REPETITIONS || 3 }} ${{ env.TESTS }} | |
| - name: Compare | |
| id: compare | |
| run: | | |
| apt-get update && apt-get install -y bc datamash --fix-missing | |
| chmod +x ./main/scripts/compare_benchmark.sh | |
| bash ./main/scripts/compare_benchmark.sh ${{ env.TESTS }} | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ql-benchmark-results | |
| path: /__w/xad/xad/ql_benchmark_results.md | |
| - name: Save the PR Number | |
| shell: bash | |
| env: | |
| PULL_REQUEST_NUMBER: ${{ github.event.number }} | |
| run: echo $PULL_REQUEST_NUMBER > /pull_request_number.txt | |
| - name: Upload the PR Number | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pull_request_number | |
| path: /__w/xad/xad/pull_request_number.txt |