integration_tests.yml

name: Integration tests

on:
  workflow_dispatch:
    inputs:
      test:
        description: the integration test to run
        default: fairscale_benchmarks
        required: true
        type: choice
        options:
          - fairscale_benchmarks
      cluster:
        description: the beaker cluster to run the test on
        default: ai2/tango-integration-tests
        required: true
        type: choice
        options:
          - ai2/tango-integration-tests
          - ai2/allennlp-cirrascale
  # Uncomment this trigger to test changes on a pull request.
  # You also have to uncomment the lines below that mention 'for pull request checks'
  # pull_request:
  #   branches:
  #     - '*'

jobs:
  run_test:
    name: ${{ github.event.inputs.test }}
    # name: fairscale_benchmarks  # for pull request checks
    runs-on: [ubuntu-latest]
    timeout-minutes: 60
    env:
      TEST_NAME: ${{ github.event.inputs.test }}
      # TEST_NAME: fairscale_benchmarks  # for pull request checks
      BEAKER_TOKEN: ${{ secrets.BEAKER_TOKEN }}
      BEAKER_WORKSPACE: ai2/tango-integration-tests
      BEAKER_CLUSTER: ${{ github.event.inputs.cluster }}
      # BEAKER_CLUSTER: ai2/allennlp-cirrascale  # for pull request checks
      IMAGE_NAME: petew/tango-testing
    steps:
      - uses: actions/checkout@v3

      - name: Validate inputs
        run: |
          # The 'test' input should be a directory in `integration_tests/`
          test -d "integration_tests/${TEST_NAME}"

      - name: Determine current commit SHA (pull request)
        if: github.event_name == 'pull_request'
        run: |
          echo "COMMIT_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV

      - name: Determine current commit SHA (push)
        if: github.event_name != 'pull_request'
        run: |
          echo "COMMIT_SHA=$GITHUB_SHA" >> $GITHUB_ENV

      - name: Install beaker client
        shell: bash
        run: |
          mkdir -p "$HOME/bin"

          # Download and install from latest GitHub release.
          curl -s https://api.github.com/repos/allenai/beaker/releases/latest \
            | grep 'browser_download_url.*linux' \
            | cut -d '"' -f 4 \
            | wget -qi - \
          && tar -xvzf beaker_linux.tar.gz -C "$HOME/bin"

          # Add to path.
          echo "$HOME/bin" >> "$GITHUB_PATH"

      - name: Verify beaker install
        run: |
          beaker account whoami

      - name: Create beaker experiment config
        run: |
          cat >beaker_config.yml << EOL
          version: v2-alpha
          description: ${{ env.TEST_NAME }}
          tasks:
            - name: test
              image:
                beaker: ${{ env.IMAGE_NAME }}
              command: ["/entrypoint.sh", "integration_tests/${{ env.TEST_NAME }}/run.sh"]
              envVars:
                - name: COMMIT_SHA
                  value: $COMMIT_SHA
                - name: WANDB_API_KEY
                  secret: WANDB_API_KEY
                - name: FILE_FRIENDLY_LOGGING
                  value: "true"
                - name: TOKENIZERS_PARALLELISM  # set this to avoid warnings
                  value: "true"
                - name: PYTHONUNBUFFERED
                  value: "true"
              result:
                path: '/results'
              resources:
                gpuCount: 4
              context:
                cluster: ${{ env.BEAKER_CLUSTER }}
                priority: normal
          EOL
          cat beaker_config.yml

      - name: Submit beaker job
        run: |
          TIMESTAMP=$(date +%H%M%S)
          EXPERIMENT=$(beaker experiment create beaker_config.yml --workspace $BEAKER_WORKSPACE --name "${TEST_NAME}-${{ github.run_number }}-${TIMESTAMP}" | awk '{print $2}')
          if [ -z "$EXPERIMENT" ]; then
            exit 1
          else
            echo "EXPERIMENT=$EXPERIMENT" >> $GITHUB_ENV
            echo "Experiment $EXPERIMENT submitted. See progress at https://beaker.org/ex/$EXPERIMENT"
          fi

      - name: Wait for job to finish
        run: |
          beaker experiment await $EXPERIMENT test finalized --timeout 60m
          # Check the job's exit code.
          test $(beaker experiment get $EXPERIMENT --format=json | jq '.[0].jobs[0].status.exitCode') -eq 0

      - name: Get logs
        if: always()
        run: |
          # EXPERIMENT could be empty if the submission step failed.
          # We'll exit right away if that's the case.
          if [ -z "$EXPERIMENT" ]; then
            echo "No logs to show"
            exit 0
          fi

          # Download logs from beaker.
          beaker experiment results $EXPERIMENT --prefix out.log --output results

          # If the experiment failed during startup, there might not be any logs.
          if [ -f results/test/out.log ]; then
            echo ""
            echo ">>> Logs:"
            echo ""
            cat results/test/out.log
          else
            echo "No logs to show"
          fi

      - name: Stop job
        if: cancelled()
        run: |
          if [ ! -z "$EXPERIMENT" ]; then
            beaker experiment stop $EXPERIMENT
          fi