From 53fb55d6c58d3b5897a53edd99ae0a86a40de5bc Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Wed, 18 Dec 2024 11:03:59 -0500 Subject: [PATCH] chore(ci): Execution time report (#6827) --- .github/workflows/reports.yml | 112 +++++++++++++++++++++++++++--- .gitignore | 1 + test_programs/execution_report.sh | 59 ++++++++++++++++ 3 files changed, 161 insertions(+), 11 deletions(-) create mode 100755 test_programs/execution_report.sh diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 0a03add8338..37864dd9b68 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -225,8 +225,8 @@ jobs: retention-days: 3 overwrite: true - generate_compilation_report: - name: Compilation time + generate_compilation_and_execution_report: + name: Compilation and execution time needs: [build-nargo] runs-on: ubuntu-22.04 permissions: @@ -253,9 +253,14 @@ jobs: working-directory: ./test_programs run: | ./compilation_report.sh - cat compilation_report.json mv compilation_report.json ../compilation_report.json + - name: Generate Execution report + working-directory: ./test_programs + run: | + ./execution_report.sh + mv execution_report.json ../execution_report.json + - name: Upload compilation report uses: actions/upload-artifact@v4 with: @@ -263,8 +268,16 @@ jobs: path: compilation_report.json retention-days: 3 overwrite: true + + - name: Upload execution report + uses: actions/upload-artifact@v4 + with: + name: in_progress_execution_report + path: execution_report.json + retention-days: 3 + overwrite: true - external_repo_compilation_report: + external_repo_compilation_and_execution_report: needs: [build-nargo] runs-on: ubuntu-latest timeout-minutes: 15 @@ -272,7 +285,7 @@ jobs: fail-fast: false matrix: include: - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, is_library: true } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail } @@ -280,7 +293,7 @@ jobs: - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public } - name: External repo compilation report - ${{ matrix.project.repo }}/${{ matrix.project.path }} + name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }} steps: - name: Download nargo binary uses: actions/download-artifact@v4 @@ -302,6 +315,13 @@ jobs: sparse-checkout: | test_programs/compilation_report.sh sparse-checkout-cone-mode: false + + - uses: actions/checkout@v4 + with: + path: scripts + sparse-checkout: | + test_programs/execution_report.sh + sparse-checkout-cone-mode: false - name: Checkout uses: actions/checkout@v4 @@ -316,9 +336,16 @@ jobs: mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh chmod +x ./compilation_report.sh ./compilation_report.sh 1 + + - name: Generate execution report + working-directory: ./test-repo/${{ matrix.project.path }} + if: ${{ !matrix.project.is_library }} + run: | + mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh + ./execution_report.sh 1 - name: Move compilation report - id: report + id: compilation_report shell: bash run: | PACKAGE_NAME=${{ matrix.project.path }} @@ -326,18 +353,36 @@ jobs: mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT + - name: Move execution report + id: execution_report + shell: bash + if: ${{ !matrix.project.is_library }} + run: | + PACKAGE_NAME=${{ matrix.project.path }} + PACKAGE_NAME=$(basename $PACKAGE_NAME) + mv ./test-repo/${{ matrix.project.path }}/execution_report.json ./execution_report_$PACKAGE_NAME.json + echo "execution_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT + - name: Upload compilation report uses: actions/upload-artifact@v4 with: - name: compilation_report_${{ steps.report.outputs.compilation_report_name }} - path: compilation_report_${{ steps.report.outputs.compilation_report_name }}.json + name: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }} + path: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}.json + retention-days: 3 + overwrite: true + + - name: Upload execution report + uses: actions/upload-artifact@v4 + with: + name: execution_report_${{ steps.execution_report.outputs.execution_report_name }} + path: execution_report_${{ steps.execution_report.outputs.execution_report_name }}.json retention-days: 3 overwrite: true upload_compilation_report: name: Upload compilation report - needs: [generate_compilation_report, external_repo_compilation_report] - # We want this job to run even if one variation of the matrix in `external_repo_compilation_report` fails + needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report] + # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails if: always() runs-on: ubuntu-latest permissions: @@ -490,3 +535,48 @@ jobs: with: header: memory message: ${{ steps.memory_report.outputs.markdown }} + + upload_execution_report: + name: Upload execution report + needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report] + # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails + if: always() + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - name: Download initial execution report + uses: actions/download-artifact@v4 + with: + name: in_progress_execution_report + + - name: Download matrix execution reports + uses: actions/download-artifact@v4 + with: + pattern: execution_report_* + path: ./reports + + - name: Merge execution reports using jq + run: | + mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh + ./merge-bench-reports.sh execution_report + + - name: Parse execution report + id: execution_report + uses: noir-lang/noir-bench-report@0954121203ee55dcda5c7397b9c669c439a20922 + with: + report: execution_report.json + header: | + # Execution Report + execution_report: true + + - name: Add memory report to sticky comment + if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: execution_time + message: ${{ steps.execution_report.outputs.markdown }} + diff --git a/.gitignore b/.gitignore index 8442d688fbf..9bfb8a89331 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ gates_report.json gates_report_brillig.json gates_report_brillig_execution.json compilation_report.json +execution_report.json # Github Actions scratch space # This gives a location to download artifacts into the repository in CI without making git dirty. diff --git a/test_programs/execution_report.sh b/test_programs/execution_report.sh new file mode 100755 index 00000000000..bdb1500d19e --- /dev/null +++ b/test_programs/execution_report.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -e + +current_dir=$(pwd) +base_path="$current_dir/execution_success" + +# Tests to be profiled for execution report +tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression") + +echo "{\"execution_reports\": [ " > $current_dir/execution_report.json + +# If there is an argument that means we want to generate a report for only the current directory +if [ "$#" -ne 0 ]; then + base_path="$current_dir" + tests_to_profile=(".") +fi + +ITER="1" +NUM_ARTIFACTS=${#tests_to_profile[@]} + +for dir in ${tests_to_profile[@]}; do + if [[ " ${excluded_dirs[@]} " =~ " ${dir} " ]]; then + continue + fi + + if [[ ${CI-false} = "true" ]] && [[ " ${ci_excluded_dirs[@]} " =~ " ${dir} " ]]; then + continue + fi + + cd $base_path/$dir + + # The default package to run is the supplied list hardcoded at the top of the script + PACKAGE_NAME=$dir + # Otherwise default to the current directory as the package we want to run + if [ "$#" -ne 0 ]; then + PACKAGE_NAME=$(basename $current_dir) + fi + + # Check whether a compilation artifact exists. + # Any programs part of this benchmark should already be compiled. + # We want to make sure that compilation time is not included in the execution time. + if [ ! -e ./target/*.json ]; then + echo "Missing compilation artifact for $PACKAGE_NAME" + exit 1 + fi + + COMPILE_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') + echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"$COMPILE_TIME\"" >> $current_dir/execution_report.json + + if (($ITER == $NUM_ARTIFACTS)); then + echo "}" >> $current_dir/execution_report.json + else + echo "}, " >> $current_dir/execution_report.json + fi + + ITER=$(( $ITER + 1 )) +done + +echo "]}" >> $current_dir/execution_report.json