benchmark.yml

name: Benchmark PR vs main

on:
  workflow_dispatch:
  pull_request:
    branches:
      - main
    paths:
    - '**.swift'
    - '**.yml'
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-apibreakage
  cancel-in-progress: true
    
jobs:
  benchmark-delta:

    runs-on: ${{ matrix.os }}
    timeout-minutes: 15
    continue-on-error: true

    strategy:
      matrix:
        os: [ubuntu-latest]

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Homebrew Mac
        if: ${{ runner.os == 'Macos' }}
        run: |
          echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH
          brew install jemalloc

      - name: Ubuntu deps
        if: ${{ runner.os == 'Linux' }}
        run: |
          sudo apt-get install -y libjemalloc-dev

      - name: Git URL token override and misc
        run: |
          #git config --global url."https://ordo-ci:${{ secrets.CI_MACHINE_PAT }}@github.com".insteadOf "https://github.com"
          #/usr/bin/ordo-performance
          [ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV
          echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH
      - name: Run benchmarks for PR branch
        if: ${{ env.hasBenchmark == '1' }}
        run: |
          cd Benchmarks
          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request
      - name: Switch to branch 'main'
        if: ${{ env.hasBenchmark == '1' }}
        run: |
          git stash
          git checkout main
      - name: Run benchmarks for branch 'main'
        if: ${{ env.hasBenchmark == '1' }}
        run: |
          cd Benchmarks
          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main
      - name: Compare PR and main
        if: ${{ env.hasBenchmark == '1' }}
        id: benchmark
        run: |
          echo $(date) >> $GITHUB_STEP_SUMMARY
          echo "exitStatus=1" >> $GITHUB_ENV
          cd Benchmarks
          swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY
          echo "exitStatus=0" >> $GITHUB_ENV
        continue-on-error: true
      - if: ${{ env.exitStatus == '0' }}
        name: Pull request comment text success
        id: prtestsuccess
        run: |
          echo 'PRTEST<<EOF' >> $GITHUB_ENV
          echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/hummingbird-project/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - if: ${{ env.exitStatus == '1' }}
        name: Pull request comment text failure
        id: prtestfailure
        run: |
          echo 'PRTEST<<EOF' >> $GITHUB_ENV
          echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/hummingbird-project/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV
          echo "_Pull request had performance regressions_" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - name: Comment PR
        if: ${{ env.hasBenchmark == '1' }}
        uses: thollander/actions-comment-pull-request@v3
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          message: ${{ env.PRTEST }}
          comment_tag: benchmark
      - name: Exit with correct status
        run: |
          exit ${{ env.exitStatus }}