leaderboard.yml

name: Manage leaderboard

on:
  issue_comment:
    types: [created]
    branches:
      - main

jobs:
  add-entry:
    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/add-to-leaderboard') }}
    runs-on: ubuntu-latest
    steps:

      - name: Obtain PR branch
        id: get-branch
        run: echo "branch=$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT
        env:
          REPO: ${{ github.repository }}
          PR_NO: ${{ github.event.issue.number }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Checkout PR branch
        uses: actions/checkout@v2
        with:
          ref: ${{ steps.get-branch.outputs.branch }}

      - name: Extract arguments from comment
        id: extract_args
        run: |
          echo "Extracting arguments..."
          comment="${{ github.event.comment.body }}"
          python_version=$(echo "$comment" | grep -oP '(?<=--python )\S+')
          appworld_version=$(echo "$comment" | grep -oP '(?<=--appworld )\S+')
          experiment_prefixes=$(echo "$comment" | cut -d' ' -f6-)

          echo "Python version: $python_version"
          echo "Appworld version: $appworld_version"
          echo "Experiment prefixes: $experiment_prefixes"

          echo "python_version=$python_version" >> $GITHUB_ENV
          echo "appworld_version=$appworld_version" >> $GITHUB_ENV
          echo "experiment_prefixes=$experiment_prefixes" >> $GITHUB_ENV

      - uses: astral-sh/setup-uv@v3
        with:
          version: "0.4.4"

      - name: Set up Python
        run: uv python install ${{ env.python_version }}

      - name: Install venv
        run: uv venv

      - name: Install dependencies
        run: |
          uv pip install appworld==${{ env.appworld_version }}
          uv run appworld install

      - name: Download appworld data
        run: uv run appworld download data

      - name: Fetch main branch
        run: git fetch origin main

      - name: Check ls
        run: ls

      - name: Check pwd
        run: pwd

      - name: Add leaderboard entry
        run: PYTHONPATH=$PYTHONPATH:. uv run scripts/add_to_leaderboard.py --pr-branch ${{ steps.get-branch.outputs.branch }} --experiment-prefixes ${{ env.experiment_prefixes }}

      - name: Comment with leaderboard entry
        if: ${{ success() }}
        uses: actions/github-script@v6
        with:
          script: |
            const fs = require('fs');
            const entries = JSON.parse(fs.readFileSync('experiments/outputs/_leaderboard.json', 'utf8'));
            const formattedEntry = '```json\n' + JSON.stringify(entries[entries.length - 1], null, 4) + '\n```';
            const commentBody = `### Added leaderboard entry\n${formattedEntry}`;
            const issue_number = context.issue.number;
            await github.rest.issues.createComment({
              ...context.repo,
              issue_number: issue_number,
              body: commentBody,
            });

      - name: Commit and push
        run: |
            git config user.name "github-actions[bot]"
            git config user.email "github-actions[bot]@users.noreply.github.com"
            git add -f experiments/outputs/_leaderboard.json
            git commit -m "add leaderboard entry"
            git push origin ${{ steps.get-branch.outputs.branch }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  remove-entry:
    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/remove-from-leaderboard') }}
    runs-on: ubuntu-latest
    steps:

      - name: Obtain PR branch
        id: get-branch
        run: echo "branch=$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT
        env:
          REPO: ${{ github.repository }}
          PR_NO: ${{ github.event.issue.number }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Checkout PR branch
        uses: actions/checkout@v2
        with:
          ref: ${{ steps.get-branch.outputs.branch }}

      - name: Extract arguments from comment
        id: extract_args
        run: |
          echo "Extracting arguments..."
          comment="${{ github.event.comment.body }}"
          ids="${comment#* }"
          echo "Entry IDs: $ids"
          echo "ids=$ids" >> $GITHUB_ENV

      - name: Remove leaderboard entries
        run: PYTHONPATH=$PYTHONPATH:. uv run scripts/remove_from_leaderboard.py --save-removed --ids ${{ env.ids }}

      - name: Reformat leaderboard in the original format
        run: |
          python - <<EOF
          import json

          with open("experiments/outputs/_leaderboard.json") as file:
            data = json.load(file)

          def normalize_numbers(obj):
            if isinstance(obj, int):
                return float(obj)
            elif isinstance(obj, dict):
                return {k: normalize_numbers(v) for k, v in obj.items()}
            elif isinstance(obj, list):
                return [normalize_numbers(v) for v in obj]
            return obj

          with open("experiments/outputs/_leaderboard.json", "w") as file:
            json.dump(normalize_numbers(data), file, indent=4)
          EOF

      - name: Commit and push
        run: |
            git config user.name "github-actions[bot]"
            git config user.email "github-actions[bot]@users.noreply.github.com"
            git add -f experiments/outputs/_leaderboard.json
            git commit -m "remove leaderboard entry"
            git push origin ${{ steps.get-branch.outputs.branch }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}