From 9ba8e2e211660cddcf472e2882549109282485b6 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Fri, 26 Jul 2024 15:26:44 +0700 Subject: [PATCH 01/20] bencher --- .github/workflows/run-benchmark.yaml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/run-benchmark.yaml index b1a0ee3a23..a4c2c6a0d1 100644 --- a/.github/workflows/run-benchmark.yaml +++ b/.github/workflows/run-benchmark.yaml @@ -1,17 +1,25 @@ name: benchmark pull requests on: + push: + branches: + - master pull_request: - types: [labeled] jobs: runBenchmark: - if: ${{ github.event.label.name == 'run-benchmark' && github.event_name == 'pull_request' }} name: run benchmark runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: jasonwilliams/criterion-compare-action@move_to_actions - with: - branchName: ${{ github.base_ref }} - token: ${{ secrets.GITHUB_TOKEN }} + - uses: bencherdev/bencher@main + - name: Run benchmarks + run: | + bencher run \ + --project nickel \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch main \ + --testbed ubuntu-latest + --adapter criterion + --err \ + cargo bench --package nickel-lang-core --bench numeric From bcbb6945cd3f6f0f7276d6496d6a1fd1d8995b97 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Fri, 26 Jul 2024 15:35:06 +0700 Subject: [PATCH 02/20] backslash --- .github/workflows/run-benchmark.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/run-benchmark.yaml index a4c2c6a0d1..7d8df5e594 100644 --- a/.github/workflows/run-benchmark.yaml +++ b/.github/workflows/run-benchmark.yaml @@ -19,7 +19,7 @@ jobs: --project nickel \ --token '${{ secrets.BENCHER_API_TOKEN }}' \ --branch main \ - --testbed ubuntu-latest - --adapter criterion + --testbed ubuntu-latest \ + --adapter criterion \ --err \ cargo bench --package nickel-lang-core --bench numeric From c23e260a826455c996845263439052e2675cbcc4 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Fri, 26 Jul 2024 15:36:30 +0700 Subject: [PATCH 03/20] rust_criterion --- .github/workflows/run-benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/run-benchmark.yaml index 7d8df5e594..b85f4bb145 100644 --- a/.github/workflows/run-benchmark.yaml +++ b/.github/workflows/run-benchmark.yaml @@ -20,6 +20,6 @@ jobs: --token '${{ secrets.BENCHER_API_TOKEN }}' \ --branch main \ --testbed ubuntu-latest \ - --adapter criterion \ + --adapter rust_criterion \ --err \ cargo bench --package nickel-lang-core --bench numeric From 616ebd1b9d089d98eea3969b1d862b01555f35b2 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Fri, 26 Jul 2024 16:02:04 +0700 Subject: [PATCH 04/20] try out the tweag runner --- .github/workflows/run-benchmark.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/run-benchmark.yaml index b85f4bb145..27619b65b9 100644 --- a/.github/workflows/run-benchmark.yaml +++ b/.github/workflows/run-benchmark.yaml @@ -7,19 +7,11 @@ on: jobs: runBenchmark: name: run benchmark - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, X64] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: bencherdev/bencher@main - name: Run benchmarks run: | - bencher run \ - --project nickel \ - --token '${{ secrets.BENCHER_API_TOKEN }}' \ - --branch main \ - --testbed ubuntu-latest \ - --adapter rust_criterion \ - --err \ cargo bench --package nickel-lang-core --bench numeric From 3825d80e65ae985dccb46396a94cde6d2062e407 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Fri, 26 Jul 2024 16:04:15 +0700 Subject: [PATCH 05/20] nix it --- .github/workflows/run-benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/run-benchmark.yaml index 27619b65b9..b6771d7aa6 100644 --- a/.github/workflows/run-benchmark.yaml +++ b/.github/workflows/run-benchmark.yaml @@ -14,4 +14,4 @@ jobs: fetch-depth: 0 - name: Run benchmarks run: | - cargo bench --package nickel-lang-core --bench numeric + nix develop --command cargo bench --package nickel-lang-core --bench numeric From 9aa226d7ddb691aea17d33f8d4736d49ec90862b Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 14:26:49 +0700 Subject: [PATCH 06/20] try using the bencher secret --- .github/workflows/run-benchmark.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/run-benchmark.yaml index b6771d7aa6..6569d7c824 100644 --- a/.github/workflows/run-benchmark.yaml +++ b/.github/workflows/run-benchmark.yaml @@ -1,17 +1,23 @@ name: benchmark pull requests on: push: - branches: - - master + branches: master pull_request: jobs: - runBenchmark: - name: run benchmark + benchmark-master: + name: benchmark master runs-on: [self-hosted, Linux, X64] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Run benchmarks + - uses: bencherdev/bencher@main + - name: Track benchmarks on master run: | + bencher run \ + --project nickel \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch master \ + --testbed build01 \ + --adapter rust_criterion \ nix develop --command cargo bench --package nickel-lang-core --bench numeric From e1cb6575d480bffa7b8671da810bd59a0ad07ef3 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 14:30:24 +0700 Subject: [PATCH 07/20] run it on ubuntu-latest --- .github/workflows/run-benchmark.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/run-benchmark.yaml index 6569d7c824..14d20497ab 100644 --- a/.github/workflows/run-benchmark.yaml +++ b/.github/workflows/run-benchmark.yaml @@ -6,11 +6,18 @@ on: jobs: benchmark-master: name: benchmark master - runs-on: [self-hosted, Linux, X64] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - uses: cachix/install-nix-action@V27 + name: "Installing Nix" + with: + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = true + nix_path: "nixpkgs=channel:nixos-unstable" - uses: bencherdev/bencher@main - name: Track benchmarks on master run: | @@ -18,6 +25,6 @@ jobs: --project nickel \ --token '${{ secrets.BENCHER_API_TOKEN }}' \ --branch master \ - --testbed build01 \ + --testbed ubuntu-latest \ --adapter rust_criterion \ nix develop --command cargo bench --package nickel-lang-core --bench numeric From c252b35bb3887101bb662d1a5d9fde33e47d47ce Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 15:45:38 +0700 Subject: [PATCH 08/20] try the master/pr split --- ...n-benchmark.yaml => benchmark-master.yaml} | 3 +- .github/workflows/benchmark-pr.yaml | 28 ++++++++ .github/workflows/upload-pr-benchmarks.yaml | 71 +++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) rename .github/workflows/{run-benchmark.yaml => benchmark-master.yaml} (94%) create mode 100644 .github/workflows/benchmark-pr.yaml create mode 100644 .github/workflows/upload-pr-benchmarks.yaml diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/benchmark-master.yaml similarity index 94% rename from .github/workflows/run-benchmark.yaml rename to .github/workflows/benchmark-master.yaml index 14d20497ab..7e3174487d 100644 --- a/.github/workflows/run-benchmark.yaml +++ b/.github/workflows/benchmark-master.yaml @@ -1,8 +1,7 @@ -name: benchmark pull requests +name: benchmark master on: push: branches: master - pull_request: jobs: benchmark-master: name: benchmark master diff --git a/.github/workflows/benchmark-pr.yaml b/.github/workflows/benchmark-pr.yaml new file mode 100644 index 0000000000..c1d5522dc2 --- /dev/null +++ b/.github/workflows/benchmark-pr.yaml @@ -0,0 +1,28 @@ +name: benchmark pull requests +on: + pull_request: + types: [opened, reopened, edited, synchronize] + +jobs: + benchmark-pr: + name: benchmark pr + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@V27 + name: "Installing Nix" + with: + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = true + nix_path: "nixpkgs=channel:nixos-unstable" + - name: run benchmarks + run: | + nix develop --command cargo bench --package nickel-lang-core --bench numeric > criterion-output.txt + - name: upload results + uses: actions/upload-artifact@v4 + with: + name: criterion-output + path: criterion-output.txt diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml new file mode 100644 index 0000000000..79eb45570d --- /dev/null +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -0,0 +1,71 @@ +name: upload pr benchmarks to bencher + +on: + workflow_run: + workflows: [benchmark pull requests] + types: [completed] + +jobs: + upload_pr_bencmhark: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + env: + BENCHMARK_RESULTS: criterion-output + PR_EVENT: event.json + steps: + - name: download results + uses: actions/github-script@v6 + with: + script: | + async function downloadArtifact(artifactName) { + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == artifactName + })[0]; + if (!matchArtifact) { + core.setFailed(`Failed to find artifact: ${artifactName}`); + } + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data)); + } + await downloadArtifact(process.env.BENCHMARK_RESULTS); + await downloadArtifact(process.env.PR_EVENT); + - name: Unzip Benchmark Results + run: | + unzip $BENCHMARK_RESULTS.zip + unzip $PR_EVENT.zip + - name: Export PR Event Data + uses: actions/github-script@v6 + with: + script: | + let fs = require('fs'); + let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'})); + core.exportVariable("PR_HEAD", `${prEvent.number}/merge`); + core.exportVariable("PR_BASE", prEvent.pull_request.base.ref); + core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha); + core.exportVariable("PR_NUMBER", prEvent.number); + - uses: bencherdev/bencher@main + - name: Track Benchmarks with Bencher + run: | + bencher run \ + --project nickel \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch '${{ env.PR_HEAD }}' \ + --branch-start-point '${{ env.PR_BASE }}' \ + --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \ + --testbed ubuntu-latest \ + --adapter rust_criterion \ + --err \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-number '${{ env.PR_NUMBER }}' \ + --file "$BENCHMARK_RESULTS" From 5856b90df8d33a0cc3b4f6fc85ef5f040d665327 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 15:58:41 +0700 Subject: [PATCH 09/20] huh? --- .github/workflows/upload-pr-benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index 79eb45570d..23ba95a57d 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -13,7 +13,7 @@ jobs: BENCHMARK_RESULTS: criterion-output PR_EVENT: event.json steps: - - name: download results + - name: download uses: actions/github-script@v6 with: script: | From 5e48f340446ec2792a65e6c6cfecd8f6ad01ff6f Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:04:18 +0700 Subject: [PATCH 10/20] Rewrite it --- .github/workflows/upload-pr-benchmarks.yaml | 120 ++++++++++---------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index 23ba95a57d..e4f7742ad2 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -1,4 +1,4 @@ -name: upload pr benchmarks to bencher +name: Track Benchmarks with Bencher on: workflow_run: @@ -6,66 +6,66 @@ on: types: [completed] jobs: - upload_pr_bencmhark: + track_fork_pr_branch: if: github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest - env: - BENCHMARK_RESULTS: criterion-output - PR_EVENT: event.json - steps: - - name: download - uses: actions/github-script@v6 - with: - script: | - async function downloadArtifact(artifactName) { - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == artifactName - })[0]; - if (!matchArtifact) { - core.setFailed(`Failed to find artifact: ${artifactName}`); + env: + BENCHMARK_RESULTS: criterion-output + PR_EVENT: event.json + steps: + - name: Download Benchmark Results + uses: actions/github-script@v6 + with: + script: | + async function downloadArtifact(artifactName) { + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == artifactName + })[0]; + if (!matchArtifact) { + core.setFailed(`Failed to find artifact: ${artifactName}`); + } + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data)); } - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); + await downloadArtifact(process.env.BENCHMARK_RESULTS); + await downloadArtifact(process.env.PR_EVENT); + - name: Unzip Benchmark Results + run: | + unzip $BENCHMARK_RESULTS.zip + unzip $PR_EVENT.zip + - name: Export PR Event Data + uses: actions/github-script@v6 + with: + script: | let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data)); - } - await downloadArtifact(process.env.BENCHMARK_RESULTS); - await downloadArtifact(process.env.PR_EVENT); - - name: Unzip Benchmark Results - run: | - unzip $BENCHMARK_RESULTS.zip - unzip $PR_EVENT.zip - - name: Export PR Event Data - uses: actions/github-script@v6 - with: - script: | - let fs = require('fs'); - let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'})); - core.exportVariable("PR_HEAD", `${prEvent.number}/merge`); - core.exportVariable("PR_BASE", prEvent.pull_request.base.ref); - core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha); - core.exportVariable("PR_NUMBER", prEvent.number); - - uses: bencherdev/bencher@main - - name: Track Benchmarks with Bencher - run: | - bencher run \ - --project nickel \ - --token '${{ secrets.BENCHER_API_TOKEN }}' \ - --branch '${{ env.PR_HEAD }}' \ - --branch-start-point '${{ env.PR_BASE }}' \ - --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \ - --testbed ubuntu-latest \ - --adapter rust_criterion \ - --err \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' \ - --ci-number '${{ env.PR_NUMBER }}' \ - --file "$BENCHMARK_RESULTS" + let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'})); + core.exportVariable("PR_HEAD", `${prEvent.number}/merge`); + core.exportVariable("PR_BASE", prEvent.pull_request.base.ref); + core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha); + core.exportVariable("PR_NUMBER", prEvent.number); + - uses: bencherdev/bencher@main + - name: Track Benchmarks with Bencher + run: | + bencher run \ + --project nickel \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch '${{ env.PR_HEAD }}' \ + --branch-start-point '${{ env.PR_BASE }}' \ + --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \ + --testbed ubuntu-latest \ + --adapter criterion_rust \ + --err \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-number '${{ env.PR_NUMBER }}' \ + --file "$BENCHMARK_RESULTS.txt" From c78820ec1f83643da0a29f551ac82503c147720a Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:18:43 +0700 Subject: [PATCH 11/20] why isn't the upload running? --- .github/workflows/upload-pr-benchmarks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index e4f7742ad2..bf4b0d8195 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -7,6 +7,7 @@ on: jobs: track_fork_pr_branch: + name: track pr benchmarks if: github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest env: From a3495fcfeecd20ccfaa826d77b0229a2b1eee864 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:21:33 +0700 Subject: [PATCH 12/20] test --- .github/workflows/upload-pr-benchmarks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index bf4b0d8195..170e9deaaa 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -6,6 +6,7 @@ on: types: [completed] jobs: +syntax error track_fork_pr_branch: name: track pr benchmarks if: github.event.workflow_run.conclusion == 'success' From 9ed782154fa8115d3f742ca3708a74973b6902ea Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:22:03 +0700 Subject: [PATCH 13/20] test --- .github/workflows/upload-pr-benchmarks.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index 170e9deaaa..bf4b0d8195 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -6,7 +6,6 @@ on: types: [completed] jobs: -syntax error track_fork_pr_branch: name: track pr benchmarks if: github.event.workflow_run.conclusion == 'success' From a8de85b4f71bcb10788f966093ac16758285d3d8 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:23:35 +0700 Subject: [PATCH 14/20] Better workflow name --- .github/workflows/release-artifacts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-artifacts.yaml b/.github/workflows/release-artifacts.yaml index d9cee164d9..b9eb3e7d19 100644 --- a/.github/workflows/release-artifacts.yaml +++ b/.github/workflows/release-artifacts.yaml @@ -1,4 +1,4 @@ -name: Upload release artifacts +name: Test Docker manifest on: release: types: [published] From 3d5964a6bec3172e4f09055ff85425e268f79ac6 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:33:47 +0700 Subject: [PATCH 15/20] test --- .github/workflows/upload-pr-benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index bf4b0d8195..24c2f2092e 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -2,7 +2,7 @@ name: Track Benchmarks with Bencher on: workflow_run: - workflows: [benchmark pull requests] + workflows: [some non-existend workflow] types: [completed] jobs: From 3a06fe540d9c2c18e12efe572e3a49e895bac111 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:41:22 +0700 Subject: [PATCH 16/20] test --- .github/workflows/upload-pr-benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index 24c2f2092e..433ef9dc65 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -2,7 +2,7 @@ name: Track Benchmarks with Bencher on: workflow_run: - workflows: [some non-existend workflow] + workflows: [Continuous integration] types: [completed] jobs: From 8159cacb02280571dd2ba9da01e72ca3e3b84dd5 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:42:53 +0700 Subject: [PATCH 17/20] test --- .github/workflows/upload-pr-benchmarks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index 433ef9dc65..f8f7f3dc2c 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -2,13 +2,13 @@ name: Track Benchmarks with Bencher on: workflow_run: - workflows: [Continuous integration] + workflows: [benchmark pull requests] types: [completed] jobs: track_fork_pr_branch: name: track pr benchmarks - if: github.event.workflow_run.conclusion == 'success' + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest env: BENCHMARK_RESULTS: criterion-output From 10f9c16a0844a6dfdc967766e1c4c3a2c4267d95 Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 16:49:15 +0700 Subject: [PATCH 18/20] test --- .github/workflows/upload-pr-benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index f8f7f3dc2c..351f4eeaa8 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -8,7 +8,7 @@ on: jobs: track_fork_pr_branch: name: track pr benchmarks - if: ${{ github.event.workflow_run.conclusion == 'success' }} + #if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest env: BENCHMARK_RESULTS: criterion-output From 72ab9b25a6e0993f5d00fe985b0c161ed972132e Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 17:04:42 +0700 Subject: [PATCH 19/20] put back the conditional --- .github/workflows/upload-pr-benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml index 351f4eeaa8..f8f7f3dc2c 100644 --- a/.github/workflows/upload-pr-benchmarks.yaml +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -8,7 +8,7 @@ on: jobs: track_fork_pr_branch: name: track pr benchmarks - #if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest env: BENCHMARK_RESULTS: criterion-output From f6fbfcef62927e6ed6c54c5152807038779b22ce Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 29 Jul 2024 20:57:40 +0700 Subject: [PATCH 20/20] Add docs --- .github/workflows/benchmark-pr.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/benchmark-pr.yaml b/.github/workflows/benchmark-pr.yaml index c1d5522dc2..73ac2a2ab7 100644 --- a/.github/workflows/benchmark-pr.yaml +++ b/.github/workflows/benchmark-pr.yaml @@ -1,3 +1,14 @@ +# This is the part of the benchmark runner that runs on pull requests. +# It's separate from the master runner for two reasons: +# - to track it differently on bencher.dev, because we only want to use master as +# the historical baseline for comparison +# - because it might want to run on PRs from forks, and those don't get access to the +# bencher API key. +# +# See the [bencher docs] for more details. +# +# [bencher docs]: https://bencher.dev/docs/how-to/github-actions/#benchmark-fork-pr-and-upload-from-default-branch + name: benchmark pull requests on: pull_request: