WASM client benchmarks #230
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run WASM client benchmarks on an instance and return parsed results to Slab CI bot. | |
| name: WASM client benchmarks | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| # Weekly benchmarks will be triggered each Saturday at 1a.m. | |
| - cron: '0 1 * * 6' | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RUST_BACKTRACE: "full" | |
| RUST_MIN_STACK: "8388608" | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| jobs: | |
| should-run: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || | |
| (github.event_name == 'push' && github.repository == 'zama-ai/tfhe-rs') | |
| permissions: | |
| pull-requests: write | |
| outputs: | |
| wasm_bench: ${{ steps.changed-files.outputs.wasm_bench_any_changed }} | |
| steps: | |
| - name: Checkout tfhe-rs | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check for file changes | |
| id: changed-files | |
| uses: tj-actions/changed-files@c3a1bb2c992d77180ae65be6ae6c166cf40f857c | |
| with: | |
| since_last_remote_commit: true | |
| files_yaml: | | |
| wasm_bench: | |
| - tfhe/Cargo.toml | |
| - concrete-csprng/** | |
| - tfhe-zk-pok/** | |
| - tfhe/src/** | |
| - '!tfhe/src/c_api/**' | |
| - tfhe/web_wasm_parallel_tests/** | |
| - .github/workflows/wasm_client_benchmark.yml | |
| setup-instance: | |
| name: Setup instance (wasm-client-benchmarks) | |
| if: github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || | |
| (github.event_name == 'push' && github.repository == 'zama-ai/tfhe-rs' && needs.should-run.outputs.wasm_bench) | |
| needs: should-run | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@801df0b8db5ea2b06128b7476c652f5ed5f193a8 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: cpu-small | |
| wasm-client-benchmarks: | |
| name: Execute WASM client benchmarks | |
| needs: setup-instance | |
| if: needs.setup-instance.result != 'skipped' | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| browser: [ chrome, firefox ] | |
| steps: | |
| - name: Checkout tfhe-rs repo with tags | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.FHE_ACTIONS_TOKEN }} | |
| - name: Get benchmark details | |
| run: | | |
| { | |
| echo "BENCH_DATE=$(date --iso-8601=seconds)"; | |
| echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; | |
| echo "COMMIT_HASH=$(git describe --tags --dirty)"; | |
| } >> "${GITHUB_ENV}" | |
| - name: Install rust | |
| uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: nightly | |
| - name: Install web resources | |
| run: | | |
| make install_node | |
| make install_${{ matrix.browser }}_browser | |
| make install_${{ matrix.browser }}_web_driver | |
| - name: Run benchmarks | |
| run: | | |
| make bench_web_js_api_parallel_${{ matrix.browser }}_ci | |
| - name: Parse results | |
| run: | | |
| make parse_wasm_benchmarks | |
| python3 ./ci/benchmark_parser.py tfhe/wasm_pk_gen.csv ${{ env.RESULTS_FILENAME }} \ | |
| --database tfhe_rs \ | |
| --hardware "m6i.4xlarge" \ | |
| --project-version "${{ env.COMMIT_HASH }}" \ | |
| --branch ${{ github.ref_name }} \ | |
| --commit-date "${{ env.COMMIT_DATE }}" \ | |
| --bench-date "${{ env.BENCH_DATE }}" \ | |
| --key-gen | |
| rm tfhe/wasm_pk_gen.csv | |
| # Run these benchmarks only once | |
| - name: Measure public key and ciphertext sizes in HL Api | |
| if: matrix.browser == 'chrome' | |
| run: | | |
| make measure_hlapi_compact_pk_ct_sizes | |
| - name: Parse key and ciphertext sizes results | |
| if: matrix.browser == 'chrome' | |
| run: | | |
| python3 ./ci/benchmark_parser.py tfhe/hlapi_cpk_and_cctl_sizes.csv ${{ env.RESULTS_FILENAME }} \ | |
| --key-gen \ | |
| --append-results | |
| - name: Upload parsed results artifact | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 | |
| with: | |
| name: ${{ github.sha }}_wasm_${{ matrix.browser }} | |
| path: ${{ env.RESULTS_FILENAME }} | |
| - name: Checkout Slab repo | |
| uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 | |
| with: | |
| repository: zama-ai/slab | |
| path: slab | |
| token: ${{ secrets.FHE_ACTIONS_TOKEN }} | |
| - name: Send data to Slab | |
| shell: bash | |
| run: | | |
| python3 slab/scripts/data_sender.py ${{ env.RESULTS_FILENAME }} "${{ secrets.JOB_SECRET }}" \ | |
| --slab-url "${{ secrets.SLAB_URL }}" | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@65e6fc1ce697e2df8149d9ae9909acc5ec5599ce | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "WASM benchmarks (${{ matrix.browser }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| teardown-instance: | |
| name: Teardown instance (wasm-client-benchmarks) | |
| if: ${{ always() && needs.setup-instance.result != 'skipped' }} | |
| needs: [ setup-instance, wasm-client-benchmarks ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@801df0b8db5ea2b06128b7476c652f5ed5f193a8 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@65e6fc1ce697e2df8149d9ae9909acc5ec5599ce | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown (wasm-client-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" |