From c99fb441652f5b2a08ca134249f2736518294792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Gustavo=20A=2E=20Amorim?= Date: Sat, 3 Feb 2024 13:51:03 -0300 Subject: [PATCH 1/4] add benchmark ci --- .github/workflows/pr_benchmarks.yml | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/pr_benchmarks.yml diff --git a/.github/workflows/pr_benchmarks.yml b/.github/workflows/pr_benchmarks.yml new file mode 100644 index 0000000000..f07d5858a4 --- /dev/null +++ b/.github/workflows/pr_benchmarks.yml @@ -0,0 +1,54 @@ +name: Benchmark (PR) + +on: + push: + branches: [test-me-*] + pull_request: + branches: [main] + types: [opened, reopened, synchronize, ready_for_review] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + +jobs: + benchmark_cpu: + name: CPU Pytest benchmark + runs-on: ubuntu-latest + + steps: + - uses: kornia/workflows/.github/actions/env@v1.5.3 + with: + fetch-depth: 25 # this is to make sure we obtain the target base commit + + - name: Setup benchmarks + run: | + echo "HEAD_JSON=$(mktemp)" >> $GITHUB_ENV + echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV + echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV + echo "RUN_BENCHMARK='pytest -vvv --benchmark-json '" >> $GITHUB_ENV + + - name: Install benchmark requirements + run: pip install -r requirements/requirements-benchmarks.txt + + - name: Run benchmarks + run: | + cd benchmarks/ + git checkout ${{ github.event.pull_request.base.sha }} + ${{ env.RUN_BENCHMARK }} ${{ env.BASE_JSON }} + git checkout ${{ github.sha }} + ${{ env.RUN_BENCHMARK }} ${{ env.HEAD_JSON }} + + - name: Comment results + uses: apbard/pytest-benchmark-commenter@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + benchmark-file: ${{ env.HEAD_JSON }} + comparison-benchmark-file: ${{ env.BASE_JSON }} + benchmark-metrics: 'name,max,mean,ops' + comparison-benchmark-metric: 'ops' + comparison-higher-is-better: true + comparison-threshold: 5 + benchmark-title: 'Result of CPU Benchmark Tests' From da695add603860d0006dcf44fffc555bd34d9aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Gustavo=20A=2E=20Amorim?= Date: Sun, 4 Feb 2024 10:44:30 -0300 Subject: [PATCH 2/4] fix run command --- .github/workflows/pr_benchmarks.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_benchmarks.yml b/.github/workflows/pr_benchmarks.yml index f07d5858a4..b87cacd443 100644 --- a/.github/workflows/pr_benchmarks.yml +++ b/.github/workflows/pr_benchmarks.yml @@ -28,7 +28,6 @@ jobs: echo "HEAD_JSON=$(mktemp)" >> $GITHUB_ENV echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV - echo "RUN_BENCHMARK='pytest -vvv --benchmark-json '" >> $GITHUB_ENV - name: Install benchmark requirements run: pip install -r requirements/requirements-benchmarks.txt @@ -37,9 +36,9 @@ jobs: run: | cd benchmarks/ git checkout ${{ github.event.pull_request.base.sha }} - ${{ env.RUN_BENCHMARK }} ${{ env.BASE_JSON }} + pytest -vvv --benchmark-json ${{ env.BASE_JSON }} git checkout ${{ github.sha }} - ${{ env.RUN_BENCHMARK }} ${{ env.HEAD_JSON }} + pytest -vvv --benchmark-json ${{ env.HEAD_JSON }} - name: Comment results uses: apbard/pytest-benchmark-commenter@v3 From e0bfc78c7a45ad30fc509888298f5fbb1b55dd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Gustavo=20A=2E=20Amorim?= Date: Sun, 4 Feb 2024 11:07:17 -0300 Subject: [PATCH 3/4] add SHA envs --- .github/workflows/pr_benchmarks.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_benchmarks.yml b/.github/workflows/pr_benchmarks.yml index b87cacd443..a156a110db 100644 --- a/.github/workflows/pr_benchmarks.yml +++ b/.github/workflows/pr_benchmarks.yml @@ -16,7 +16,7 @@ concurrency: jobs: benchmark_cpu: name: CPU Pytest benchmark - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: kornia/workflows/.github/actions/env@v1.5.3 @@ -25,6 +25,8 @@ jobs: - name: Setup benchmarks run: | + echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV + echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV echo "HEAD_JSON=$(mktemp)" >> $GITHUB_ENV echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV @@ -36,9 +38,9 @@ jobs: run: | cd benchmarks/ git checkout ${{ github.event.pull_request.base.sha }} - pytest -vvv --benchmark-json ${{ env.BASE_JSON }} + pytest ./ -vvv --benchmark-json ${{ env.BASE_JSON }} git checkout ${{ github.sha }} - pytest -vvv --benchmark-json ${{ env.HEAD_JSON }} + pytest ./ -vvv --benchmark-json ${{ env.HEAD_JSON }} - name: Comment results uses: apbard/pytest-benchmark-commenter@v3 From 2763518d5807457d9b160811c0fb4fc6675ea27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Gustavo=20A=2E=20Amorim?= Date: Sun, 4 Feb 2024 19:11:55 -0300 Subject: [PATCH 4/4] move to benchmark-action/github-action-benchmar --- .github/workflows/pr_benchmarks.yml | 41 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr_benchmarks.yml b/.github/workflows/pr_benchmarks.yml index a156a110db..190dc22bd5 100644 --- a/.github/workflows/pr_benchmarks.yml +++ b/.github/workflows/pr_benchmarks.yml @@ -15,8 +15,12 @@ concurrency: jobs: benchmark_cpu: + # NOTE: from https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#stability-of-virtual-environment + #As far as watching the benchmark results of examples in this repository, the amplitude of the benchmarks + #is about +- 10~20%. If your benchmarks use some resources such as networks or file I/O, the amplitude + #might be bigger. name: CPU Pytest benchmark - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: kornia/workflows/.github/actions/env@v1.5.3 @@ -25,8 +29,6 @@ jobs: - name: Setup benchmarks run: | - echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV - echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV echo "HEAD_JSON=$(mktemp)" >> $GITHUB_ENV echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV @@ -34,22 +36,33 @@ jobs: - name: Install benchmark requirements run: pip install -r requirements/requirements-benchmarks.txt - - name: Run benchmarks + - name: Run benchmarks BASE + # TODO: Save it using actions/cache then we don't need to regenerate it. + # By caching the result, it will also use the same information across PR's run: | cd benchmarks/ git checkout ${{ github.event.pull_request.base.sha }} pytest ./ -vvv --benchmark-json ${{ env.BASE_JSON }} + + - name: Run benchmarks HEAD + run: | + cd benchmarks/ git checkout ${{ github.sha }} pytest ./ -vvv --benchmark-json ${{ env.HEAD_JSON }} - - name: Comment results - uses: apbard/pytest-benchmark-commenter@v3 + - name: Comment benchmark result + uses: benchmark-action/github-action-benchmark@v1 with: - token: ${{ secrets.GITHUB_TOKEN }} - benchmark-file: ${{ env.HEAD_JSON }} - comparison-benchmark-file: ${{ env.BASE_JSON }} - benchmark-metrics: 'name,max,mean,ops' - comparison-benchmark-metric: 'ops' - comparison-higher-is-better: true - comparison-threshold: 5 - benchmark-title: 'Result of CPU Benchmark Tests' + tool: "pytest" + ref: ${{ github.sha }} + output-file-path: ${{ env.HEAD_JSON }} + external-data-json-path: ${{ env.BASE_JSON }} + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + comment-on-alert: true + fail-on-alert: true + summary-always: true + skip-fetch-gh-pages: true + auto-push: false + save-data-file: false + # alert-comment-cc-users: '@johnnv1 @edgarriba'