cpu_local_marqo.yml

name: cpu_local_marqo_API_tests
# runs API Tests on CPU-only instance with Vespa and Marqo running in two separate docker containers

on:
  workflow_call:
  workflow_dispatch:
    inputs:
      py_marqo_branch:
        required: false
        default: mainline
        description: >
          The "py-marqo" branch this test is running against.
          This is optional. If left as the default value "mainline", we run the test based on the "mainline" branch of
          py-marqo. Otherwise, the specified branch is tested. For example "prefix/test-xx". You can also use "marqo" and 
          it will use the latest pypi release.
      image_identifier:
        required: false
        # This is the name of the docker image that is built by the build script:
        default: marqo_docker_0
        description: >
          This is optional. If left as the default value "marqo_docker_0", the docker image built from this branch is tested.
          Otherwise, the specified Docker image is tested. For example "marqoai/marqo:test"
  push:
    branches:
      - mainline
      - releases/*
    paths-ignore:
      - '**.md'
  pull_request:
    branches:
      - mainline
      - releases/*
    paths-ignore:
      - '**.md'

permissions:
  contents: read

concurrency:
  group: cpu-local-api-tests-${{ github.ref }}
  cancel-in-progress: true

jobs:
  Start-Runner:
    name: Start self-hosted EC2 runner
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.MARQO_WORKFLOW_TESTS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.MARQO_WORKFLOW_TESTS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1
      - name: Start EC2 runner
        id: start-ec2-runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          ec2-image-id: ${{ vars.MARQO_CPU_AMD64_TESTS_INSTANCE_AMI }}
          ec2-instance-type: m6i.xlarge
          subnet-id: ${{ secrets.MARQO_WORKFLOW_TESTS_SUBNET_ID }}
          security-group-id: ${{ secrets.MARQO_WORKFLOW_TESTS_SECURITY_GROUP_ID }}
          aws-resource-tags: > # optional, requires additional permissions
            [
              {"Key": "Name", "Value": "marqo-github-runner-${{ github.run_id }}"},
              {"Key": "GitHubRepo", "Value": "${{ github.repository }}"},
              {"Key": "WorkflowName", "Value": "${{ github.workflow }}"},
              {"Key": "WorkflowRunId", "Value": "${{ github.run_id }}"},
              {"Key": "WorlflowURL", "Value": "${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}"},
              {"Key": "PoloRole", "Value": "testing"}
            ]

  Test-Marqo:
    name: Run CPU Local Marqo API Tests
    needs: Start-Runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
    defaults:
      run:
        working-directory: tests/api_tests/v1
    environment: marqo-test-suite

    steps:

      - name: Checkout Marqo repo
        uses: actions/checkout@v3

      - name: Set up Python 3.9
        uses: actions/setup-python@v3
        with:
          python-version: "3.9"
          cache: "pip"

      - name: Install Dependencies
        run: pip install -r requirements.txt

      - name: Install py-marqo
        run: bash scripts/install_pymarqo.sh ${{ inputs.py_marqo_branch }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Run Integration Tests - CPU Local Marqo
        run: |
          CUSTOM_TEST_IMG="${{ github.event.inputs.image_identifier }}"
          export TESTING_CONFIGURATION=CPU_LOCAL_MARQO
          export MARQO_API_TESTS_ROOT=$(pwd)
          export MARQO_IMAGE_NAME=${CUSTOM_TEST_IMG:-"marqo_docker_0"}
          
          bash scripts/build_marqo.sh $MARQO_IMAGE_NAME

          bash scripts/start_local_marqo.sh $MARQO_IMAGE_NAME
          
          pytest tests/api_tests/test_create_index.py

  Stop-Runner:
    name: Stop self-hosted EC2 runner
    needs:
      - Start-Runner # required to get output from the start-runner job
      - Test-Marqo # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.MARQO_WORKFLOW_TESTS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.MARQO_WORKFLOW_TESTS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}