bench.yml

name: Benchmark

defaults:
  run:
    shell: bash

# See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
concurrency:
  group: ${{ github.head_ref }}-${{ github.workflow }}
  cancel-in-progress: true

on:
  pull_request:
    branches:
      - '**'

jobs:
  pre_job:
    runs-on: ubuntu-latest
    if: contains(github.event.pull_request.labels.*.name, 'performance')
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
      - id: skip_check
        uses: fkirc/skip-duplicate-actions@v5.3.1
        with:
          cancel_others: false
          paths_ignore: '[ "**/docs/**"
                         , "**.md"
                         , "**/LICENSE"
                         , ".circleci/**"
                         , "**.nix"
                         , "**/test/**"
                         , "flake.lock"
                         , "**/README.md"
                         , "FUNDING.yml"
                         , "**/stack*.yaml"
                         , ".gitlab-ci.yaml"
                         , ".gitlab/**"
                         ]'

  bench_init:
    if: needs.pre_job.outputs.should_skip != 'true'
    needs: pre_job
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        # benching the two latest GHCs we support now
        # since benchmark are expansive.
        # choosing the two latest are easier to maintain and more forward looking
        # see discussion https://github.com/haskell/haskell-language-server/pull/4118
        # also possible to add more GHCs if we performs better in the future.
        ghc:
          - '9.6'
          - '9.8'
        os:
          - ubuntu-latest

    # This code is fitted to the strategy: assumes Linux is used ... etc,
    # change of the strategy may require changing the bootstrapping/run code

    steps:
    - uses: actions/checkout@v3
      with:

        # By default, the `pull_request` event has a `GITHUB_SHA` env variable
        # set to the "last merge commit on the GITHUB_REF branch" (see
        # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request).
        # But we want to check out the latest commit on the branch whether or
        # not it is a merge commit, so this is how we do that.
        ref: "${{ github.event.pull_request.head.sha }}"

    - run: git fetch origin master # check the master branch for benchmarking

    - uses: ./.github/actions/setup-build
      with:
        ghc: ${{ matrix.ghc }}
        os:  ${{ runner.os }}
        shorten-hls: "false"

    # max-backjumps is increased as a temporary solution
    # for dependency resolution failure
    - run: cabal configure --enable-benchmarks --max-backjumps 12000

    - name: Build
      run: cabal build haskell-language-server:benchmark

    - name: Bench init
      run: cabal bench -j --benchmark-options="all-binaries"

    # tar is required to preserve file permissions
    # compression speeds up upload/download nicely
    - name: tar workspace
      run: tar -czf workspace.tar.gz * .git

    - name: tar cabal
      run: |
        cd ~/.cabal
        tar -czf cabal.tar.gz *

    - name: Upload workspace
      uses: actions/upload-artifact@v3
      with:
        name: workspace-${{ matrix.ghc }}-${{ matrix.os }}
        retention-days: 1
        path: workspace.tar.gz

    - name: Upload .cabal
      uses: actions/upload-artifact@v3
      with:
        name: cabal-home-${{ matrix.ghc }}-${{ matrix.os }}
        retention-days: 1
        path: ~/.cabal/cabal.tar.gz

  bench_example:
    needs: [bench_init, pre_job]
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        ghc: ['9.6', '9.8']
        os: [ubuntu-latest]
        cabal: ['3.10']
        example: ['cabal', 'lsp-types']

    steps:
    - uses: haskell-actions/setup@v2.7.6
      with:
        ghc-version  : ${{ matrix.ghc   }}
        cabal-version: ${{ matrix.cabal }}
        enable-stack: false

    - name: Download cabal home
      uses: actions/download-artifact@v3
      with:
        name: cabal-home-${{ matrix.ghc }}-${{ matrix.os }}
        path: .

    - name: Download workspace
      uses: actions/download-artifact@v3
      with:
        name: workspace-${{ matrix.ghc }}-${{ matrix.os }}
        path: .

    - name: untar
      run: |
        mkdir -p ~/.cabal
        tar xzf workspace.tar.gz
        tar xzf cabal.tar.gz --directory ~/.cabal

    - name: Bench
      run: cabal bench -j --benchmark-options="${{ matrix.example }}"

    - name: Display results
      run: |
        column -s, -t < bench-results/unprofiled/${{ matrix.example }}/results.csv | tee bench-results/unprofiled/${{ matrix.example }}/results.txt
        echo
        echo "Performance Diff(comparing to its previous Version):"
        column -s, -t < bench-results/unprofiled/${{ matrix.example }}/resultDiff.csv | tee bench-results/unprofiled/${{ matrix.example }}/resultDiff.txt

    - name: tar benchmarking artifacts
      run:  find bench-results -name "*.csv" -or -name "*.svg" -or -name "*.html" | xargs tar -czf benchmark-artifacts.tar.gz

    - name: Archive benchmarking artifacts
      uses: actions/upload-artifact@v3
      with:
        name: bench-results-${{ matrix.example }}-${{ runner.os }}-${{ matrix.ghc }}
        path: benchmark-artifacts.tar.gz

    - name: tar benchmarking logs
      # We dont' store the eventlogs because the CI workers risk running out of disk space
      run: find bench-results -name "*.log" -or -name "*.hp" | xargs tar -czf benchmark-logs.tar.gz

    - name: Archive benchmark logs
      uses: actions/upload-artifact@v3
      with:
        name: bench-logs-${{ matrix.example }}-${{ runner.os }}-${{ matrix.ghc }}
        path: benchmark-logs.tar.gz

  bench_post_job:
    if: always()
    runs-on: ubuntu-latest
    needs: [pre_job, bench_init, bench_example]
    steps:
      - run: |
          echo "jobs info: ${{ toJSON(needs) }}"
      - if: contains(needs.*.result, 'failure')
        run: exit 1
      - if: contains(needs.*.result, 'cancelled') && needs.pre_job.outputs.should_skip != 'true'
        run: exit 1