From 8da31dbbede906bac512c4d156f4fcf87366c959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Test=C3=A9?= Date: Tue, 8 Oct 2024 09:47:19 +0200 Subject: [PATCH] chore(ci): run erc20 benchmarks in github It also send results to zama's benchmark database. --- .github/workflows/benchmark_erc20.yml | 144 ++++++++++++++++++++++++++ Makefile | 6 ++ tfhe/benches/high_level_api/erc20.rs | 127 +++++++++++++++++++---- 3 files changed, 254 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/benchmark_erc20.yml diff --git a/.github/workflows/benchmark_erc20.yml b/.github/workflows/benchmark_erc20.yml new file mode 100644 index 0000000000..668ccb4a01 --- /dev/null +++ b/.github/workflows/benchmark_erc20.yml @@ -0,0 +1,144 @@ +# Run all ERC20 benchmarks on an AWS instance and return parsed results to Slab CI bot. +name: ERC20 benchmarks + +on: + workflow_dispatch: + schedule: + # Weekly benchmarks will be triggered each Saturday at 5a.m. + - cron: '0 5 * * 6' + +env: + CARGO_TERM_COLOR: always + RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUST_BACKTRACE: "full" + RUST_MIN_STACK: "8388608" + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +jobs: + setup-instance: + name: Setup instance (erc20-benchmarks) + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: bench + + erc20-benchmarks: + name: Execute ERC20 benchmarks + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + concurrency: + group: ${{ github.workflow }}_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + continue-on-error: true + timeout-minutes: 720 # 12 hours + steps: + - name: Checkout tfhe-rs repo with tags + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 + with: + fetch-depth: 0 + token: ${{ secrets.FHE_ACTIONS_TOKEN }} + + - name: Get benchmark details + run: | + { + echo "BENCH_DATE=$(date --iso-8601=seconds)"; + echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; + echo "COMMIT_HASH=$(git describe --tags --dirty)"; + } >> "${GITHUB_ENV}" + + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + + - name: Install rust + uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a + with: + toolchain: nightly + + - name: Checkout Slab repo + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.FHE_ACTIONS_TOKEN }} + + - name: Run benchmarks + run: | + make bench_hlapi_erc20 + + - name: Parse results + run: | + python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ + --database tfhe_rs \ + --hardware "hpc7a.96xlarge" \ + --project-version "${{ env.COMMIT_HASH }}" \ + --branch ${{ github.ref_name }} \ + --commit-date "${{ env.COMMIT_DATE }}" \ + --bench-date "${{ env.BENCH_DATE }}" \ + --walk-subdirs \ + --name-suffix avx512 + + - name: Parse PBS counts + run: | + python3 ./ci/benchmark_parser.py tfhe/erc20_pbs_count.csv ${{ env.RESULTS_FILENAME }} \ + --key-sizes \ + --append-results + + - name: Upload parsed results artifact + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 + with: + name: ${{ github.sha }}_erc20 + path: ${{ env.RESULTS_FILENAME }} + + - name: Send data to Slab + shell: bash + run: | + python3 slab/scripts/data_sender.py ${{ env.RESULTS_FILENAME }} "${{ secrets.JOB_SECRET }}" \ + --slab-url "${{ secrets.SLAB_URL }}" + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "ERC20 benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + name: Teardown instance (erc20-benchmarks) + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, erc20-benchmarks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown (erc20-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/Makefile b/Makefile index ea73432f11..900f7a475e 100644 --- a/Makefile +++ b/Makefile @@ -1149,6 +1149,12 @@ bench_web_js_api_parallel_firefox_ci: setup_venv nvm use $(NODE_VERSION) && \ $(MAKE) bench_web_js_api_parallel_firefox +.PHONY: bench_hlapi_erc20 # Run benchmarks for ECR20 operations +bench_hlapi_erc20: install_rs_check_toolchain + RUSTFLAGS="$(RUSTFLAGS)" cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ + --bench hlapi-erc20 \ + --features=$(TARGET_ARCH_FEATURE),integer,internal-keycache,pbs-stats,nightly-avx512 -p $(TFHE_SPEC) -- + # # Utility tools # diff --git a/tfhe/benches/high_level_api/erc20.rs b/tfhe/benches/high_level_api/erc20.rs index c94f235716..75aea89543 100644 --- a/tfhe/benches/high_level_api/erc20.rs +++ b/tfhe/benches/high_level_api/erc20.rs @@ -1,9 +1,14 @@ +#[path = "../utilities.rs"] +mod utilities; + +use crate::utilities::{write_to_json, OperatorType}; use criterion::measurement::WallTime; use criterion::{BenchmarkGroup, Criterion, Throughput}; use rand::prelude::*; use rand::thread_rng; use rayon::prelude::*; use std::ops::{Add, Mul, Sub}; +use tfhe::keycache::NamedParam; use tfhe::prelude::*; use tfhe::shortint::parameters::*; use tfhe::{set_server_key, ClientKey, CompressedServerKey, ConfigBuilder, FheBool, FheUint64}; @@ -103,31 +108,69 @@ where } #[cfg(feature = "pbs-stats")] -fn print_transfer_pbs_counts( - client_key: &ClientKey, - type_name: &str, - fn_name: &str, - transfer_func: F, -) where - FheType: FheEncrypt, - F: for<'a> Fn(&'a FheType, &'a FheType, &'a FheType) -> (FheType, FheType), -{ - let mut rng = thread_rng(); +mod pbs_stats { + use super::*; + use std::fs::{File, OpenOptions}; + use std::io::Write; + use std::path::Path; + + fn write_result(file: &mut File, name: &str, value: usize) { + let line = format!("{name},{value}\n"); + let error_message = format!("cannot write {name} result into file"); + file.write_all(line.as_bytes()).expect(&error_message); + } - let from_amount = FheType::encrypt(rng.gen::(), client_key); - let to_amount = FheType::encrypt(rng.gen::(), client_key); - let amount = FheType::encrypt(rng.gen::(), client_key); + pub fn print_transfer_pbs_counts( + client_key: &ClientKey, + type_name: &str, + fn_name: &str, + transfer_func: F, + ) where + FheType: FheEncrypt, + F: for<'a> Fn(&'a FheType, &'a FheType, &'a FheType) -> (FheType, FheType), + { + let mut rng = thread_rng(); - tfhe::reset_pbs_count(); - let (_, _) = transfer_func(&from_amount, &to_amount, &amount); - let count = tfhe::get_pbs_count(); + let from_amount = FheType::encrypt(rng.gen::(), client_key); + let to_amount = FheType::encrypt(rng.gen::(), client_key); + let amount = FheType::encrypt(rng.gen::(), client_key); - println!("ERC20 transfer/{fn_name}::{type_name}: {count} PBS"); + tfhe::reset_pbs_count(); + let (_, _) = transfer_func(&from_amount, &to_amount, &amount); + let count = tfhe::get_pbs_count(); + + println!("ERC20 transfer/{fn_name}::{type_name}: {count} PBS"); + + let params = client_key.computation_parameters(); + let test_name = format!("hlapi::erc20::pbs_count::{fn_name}::{type_name}"); + + let results_file = Path::new("erc20_pbs_count.csv"); + if !results_file.exists() { + File::create(results_file).expect("create results file failed"); + } + let mut file = OpenOptions::new() + .append(true) + .open(results_file) + .expect("cannot open results file"); + + write_result(&mut file, &test_name, count as usize); + + write_to_json::( + &test_name, + params, + params.name(), + "pbs-count", + &OperatorType::Atomic, + 0, + vec![], + ); + } } fn bench_transfer_latency( c: &mut BenchmarkGroup<'_, WallTime>, client_key: &ClientKey, + bench_name: &str, type_name: &str, fn_name: &str, transfer_func: F, @@ -135,8 +178,8 @@ fn bench_transfer_latency( FheType: FheEncrypt, F: for<'a> Fn(&'a FheType, &'a FheType, &'a FheType) -> (FheType, FheType), { - let id_name = format!("{fn_name}::{type_name}"); - c.bench_function(&id_name, |b| { + let bench_id = format!("{bench_name}::{fn_name}::{type_name}"); + c.bench_function(&bench_id, |b| { let mut rng = thread_rng(); let from_amount = FheType::encrypt(rng.gen::(), client_key); @@ -147,11 +190,24 @@ fn bench_transfer_latency( let (_, _) = transfer_func(&from_amount, &to_amount, &amount); }) }); + + let params = client_key.computation_parameters(); + + write_to_json::( + &bench_id, + params, + params.name(), + "erc20-transfer", + &OperatorType::Atomic, + 64, + vec![], + ); } fn bench_transfer_throughput( group: &mut BenchmarkGroup<'_, WallTime>, client_key: &ClientKey, + bench_name: &str, type_name: &str, fn_name: &str, transfer_func: F, @@ -163,8 +219,8 @@ fn bench_transfer_throughput( for num_elems in [10, 100, 500] { group.throughput(Throughput::Elements(num_elems)); - let id_name = format!("{fn_name}::{type_name}::{num_elems}"); - group.bench_with_input(id_name, &num_elems, |b, &num_elems| { + let bench_id = format!("{bench_name}::{fn_name}::{type_name}::{num_elems}_elems"); + group.bench_with_input(&bench_id, &num_elems, |b, &num_elems| { let from_amounts = (0..num_elems) .map(|_| FheType::encrypt(rng.gen::(), client_key)) .collect::>(); @@ -184,9 +240,24 @@ fn bench_transfer_throughput( }) }) }); + + let params = client_key.computation_parameters(); + + write_to_json::( + &bench_id, + params, + params.name(), + "erc20-transfer", + &OperatorType::Atomic, + 64, + vec![], + ); } } +#[cfg(feature = "pbs-stats")] +use pbs_stats::print_transfer_pbs_counts; + fn main() { #[cfg(not(feature = "gpu"))] let params = PARAM_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M64; @@ -230,10 +301,12 @@ fn main() { // FheUint64 latency { - let mut group = c.benchmark_group("ERC20 latency"); + let bench_name = "hlapi::erc20::transfer_latency"; + let mut group = c.benchmark_group(bench_name); bench_transfer_latency( &mut group, &cks, + bench_name, "FheUint64", "whitepaper", transfer_whitepaper::, @@ -241,6 +314,7 @@ fn main() { bench_transfer_latency( &mut group, &cks, + bench_name, "FheUint64", "no_cmux", transfer_no_cmux::, @@ -248,6 +322,7 @@ fn main() { bench_transfer_latency( &mut group, &cks, + bench_name, "FheUint64", "overflow", transfer_overflow::, @@ -255,6 +330,7 @@ fn main() { bench_transfer_latency( &mut group, &cks, + bench_name, "FheUint64", "safe", transfer_safe::, @@ -265,10 +341,12 @@ fn main() { // FheUint64 Throughput { - let mut group = c.benchmark_group("ERC20 throughput"); + let bench_name = "hlapi::erc20::transfer_throughput"; + let mut group = c.benchmark_group(bench_name); bench_transfer_throughput( &mut group, &cks, + bench_name, "FheUint64", "whitepaper", transfer_whitepaper::, @@ -276,6 +354,7 @@ fn main() { bench_transfer_throughput( &mut group, &cks, + bench_name, "FheUint64", "no_cmux", transfer_no_cmux::, @@ -283,6 +362,7 @@ fn main() { bench_transfer_throughput( &mut group, &cks, + bench_name, "FheUint64", "overflow", transfer_overflow::, @@ -290,6 +370,7 @@ fn main() { bench_transfer_throughput( &mut group, &cks, + bench_name, "FheUint64", "safe", transfer_safe::,