benchmarks.yml

name: Performance benchmarks

on:
  pull_request:
    # types: [opened, ready_for_review]
    branches:
      - main
  issue_comment:
    types: [created]

jobs:
  run-benchmarks:
    if: >
      github.event_name == 'pull_request' || 
      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/rerun-benchmarks'))
    runs-on: ubuntu-latest
    steps:
      - name: Checkout PR branch
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Add project directory to PYTHONPATH
        run: echo "PYTHONPATH=$PYTHONPATH:$(pwd)" >> $GITHUB_ENV
      - name: Install dependencies
        run: pip install numpy pandas tqdm tabulate
#      - name: Checkout main branch
#        uses: actions/checkout@v4
#        with:
#          ref: main
#          repository: projectmesa/mesa
      - name: Run benchmarks on main branch
        working-directory: benchmarks
        run: python global_benchmark.py
      - name: Upload benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: timings-main
          path: benchmarks/timings_1.pickle
      - name: Checkout PR branch
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          clean: false
      - name: Download benchmark results
        uses: actions/download-artifact@v4
        with:
          name: timings-main
          path: benchmarks
      - name: Run benchmarks on PR branch
        working-directory: benchmarks
        run: python global_benchmark.py
      - name: Run compare timings
        working-directory: benchmarks
        run: |
          echo "TIMING_COMPARISON=$(python compare_timings.py)" >> $GITHUB_ENV
      - name: Run compare timings and set output
        run: |
          TIMING_COMPARISON=$(python compare_timings.py | base64)
          echo "TIMING_COMPARISON=$TIMING_COMPARISON" >> $GITHUB_ENV

      - name: Comment PR
        uses: actions/github-script@v5
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const output = Buffer.from(process.env.TIMING_COMPARISON, 'base64').toString('utf-8');
            const issue_number = context.issue.number;
            github.rest.issues.createComment({
              issue_number: issue_number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: 'Benchmark Comparison:\n' + output
            });