pr-integration-tests.yml

name: Run Integration Tests v2
concurrency:
  group: Run-Integration-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
  cancel-in-progress: true

on:
  merge_group:
  pull_request:
    branches:
      - main
      - "release/**"

env:
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
  CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
  CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
  CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}

jobs:
  build-images:
    # See https://runs-on.com/runners/linux/
    runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_TOKEN }}

      # tag every docker image with "test" so that we can spin up the correct set
      # of images during testing

      # We don't need to build the Web Docker image since it's not yet used
      # in the integration tests. We have a separate action to verify that it builds
      # successfully.
      - name: Pull Web Docker image
        run: |
          docker pull onyxdotapp/onyx-web-server:latest
          docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test

      # we use the runs-on cache for docker builds
      # in conjunction with runs-on runners, it has better speed and unlimited caching
      # https://runs-on.com/caching/s3-cache-for-github-actions/
      # https://runs-on.com/caching/docker/
      # https://github.com/moby/buildkit#s3-cache-experimental

      # images are built and run locally for testing purposes. Not pushed.
      - name: Build Backend Docker image
        uses: ./.github/actions/custom-build-and-push
        with:
          context: ./backend
          file: ./backend/Dockerfile
          platforms: linux/amd64
          tags: onyxdotapp/onyx-backend:test
          push: false
          load: true
          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max

      - name: Build Model Server Docker image
        uses: ./.github/actions/custom-build-and-push
        with:
          context: ./backend
          file: ./backend/Dockerfile.model_server
          platforms: linux/amd64
          tags: onyxdotapp/onyx-model-server:test
          push: false
          load: true
          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max

      - name: Build integration test Docker image
        uses: ./.github/actions/custom-build-and-push
        with:
          context: ./backend
          file: ./backend/tests/integration/Dockerfile
          platforms: linux/amd64
          tags: onyxdotapp/onyx-integration:test
          push: false
          load: true
          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max

      - name: Save and compress Docker images
        run: |
          # Save and compress images in parallel
          docker save onyxdotapp/onyx-web-server:test | gzip > web-server.tar.gz &
          docker save onyxdotapp/onyx-backend:test | gzip > backend.tar.gz &
          docker save onyxdotapp/onyx-model-server:test | gzip > model-server.tar.gz &
          docker save onyxdotapp/onyx-integration:test | gzip > integration.tar.gz &
          wait

      - name: Upload Docker images
        uses: actions/upload-artifact@v4
        with:
          name: docker-images
          compression-level: 9  # Maximum compression
          retention-days: 1     # Minimize storage
          path: |
            web-server.tar.gz
            backend.tar.gz
            model-server.tar.gz
            integration.tar.gz

  multi-tenant-tests:
    needs: build-images
    runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download Docker images
        uses: actions/download-artifact@v4
        with:
          name: docker-images

      - name: Load Docker images
        run: |
          # Decompress and load images in parallel
          gunzip -c web-server.tar.gz | docker load &
          gunzip -c backend.tar.gz | docker load &
          gunzip -c model-server.tar.gz | docker load &
          gunzip -c integration.tar.gz | docker load &
          wait

      - name: Start Docker containers for multi-tenant tests
        run: |
          cd deployment/docker_compose
          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
          MULTI_TENANT=true \
          AUTH_TYPE=basic \
          REQUIRE_EMAIL_VERIFICATION=false \
          DISABLE_TELEMETRY=true \
          IMAGE_TAG=test \
          docker compose -f docker-compose.dev.yml -p danswer-stack up -d
        id: start_docker_multi_tenant

      - name: Wait for services to be ready
        uses: ./.github/actions/wait-for-services

      - name: Run Multi-Tenant Integration Tests
        run: |
          echo "Running multi-tenant integration tests..."
          docker run --rm --network danswer-stack_default \
            --name test-runner \
            -e POSTGRES_HOST=relational_db \
            -e POSTGRES_USER=postgres \
            -e POSTGRES_PASSWORD=password \
            -e POSTGRES_DB=postgres \
            -e VESPA_HOST=index \
            -e REDIS_HOST=cache \
            -e API_SERVER_HOST=api_server \
            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
            -e TEST_WEB_HOSTNAME=test-runner \
            -e AUTH_TYPE=cloud \
            -e MULTI_TENANT=true \
            -e PYTEST_ADDOPTS="-n auto --dist=loadscope" \
            onyxdotapp/onyx-integration:test \
            pytest /app/tests/integration/multitenant_tests
        id: run_multitenant_tests

      - name: Save Docker logs
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log
          mv docker-compose.log ${{ github.workspace }}/docker-compose.log

      - name: Upload logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: docker-logs
          path: ${{ github.workspace }}/docker-compose.log

      - name: Stop Docker containers
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p danswer-stack down -v

  standard-tests:
    needs: build-images
    runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download Docker images
        uses: actions/download-artifact@v4
        with:
          name: docker-images

      - name: Load Docker images
        run: |
          # Decompress and load images in parallel
          gunzip -c web-server.tar.gz | docker load &
          gunzip -c backend.tar.gz | docker load &
          gunzip -c model-server.tar.gz | docker load &
          gunzip -c integration.tar.gz | docker load &
          wait

      - name: Start Docker containers
        run: |
          cd deployment/docker_compose
          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
          AUTH_TYPE=basic \
          REQUIRE_EMAIL_VERIFICATION=false \
          DISABLE_TELEMETRY=true \
          IMAGE_TAG=test \
          docker compose -f docker-compose.dev.yml -p danswer-stack up -d
        id: start_docker

      - name: Wait for services to be ready
        uses: ./.github/actions/wait-for-services

      - name: Run Standard Integration Tests
        run: |
          echo "Running standard integration tests..."
          docker run --rm --network danswer-stack_default \
            --name test-runner \
            -e POSTGRES_HOST=relational_db \
            -e POSTGRES_USER=postgres \
            -e POSTGRES_PASSWORD=password \
            -e POSTGRES_DB=postgres \
            -e VESPA_HOST=index \
            -e REDIS_HOST=cache \
            -e API_SERVER_HOST=api_server \
            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
            -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
            -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
            -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
            -e TEST_WEB_HOSTNAME=test-runner \
            -e PYTEST_ADDOPTS="-n auto --dist=loadscope" \
            onyxdotapp/onyx-integration:test \
            pytest /app/tests/integration/tests
        id: run_standard_tests

      - name: Save Docker logs
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log
          mv docker-compose.log ${{ github.workspace }}/docker-compose.log

      - name: Upload logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: docker-logs
          path: ${{ github.workspace }}/docker-compose.log

      - name: Stop Docker containers
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p danswer-stack down -v

  connector-tests:
    needs: build-images
    runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download Docker images
        uses: actions/download-artifact@v4
        with:
          name: docker-images

      - name: Load Docker images
        run: |
          # Decompress and load images in parallel
          gunzip -c web-server.tar.gz | docker load &
          gunzip -c backend.tar.gz | docker load &
          gunzip -c model-server.tar.gz | docker load &
          gunzip -c integration.tar.gz | docker load &
          wait

      - name: Start Docker containers
        run: |
          cd deployment/docker_compose
          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
          AUTH_TYPE=basic \
          REQUIRE_EMAIL_VERIFICATION=false \
          DISABLE_TELEMETRY=true \
          IMAGE_TAG=test \
          docker compose -f docker-compose.dev.yml -p danswer-stack up -d
        id: start_docker

      - name: Wait for services to be ready
        uses: ./.github/actions/wait-for-services

      - name: Run Connector Integration Tests
        run: |
          echo "Running connector integration tests..."
          docker run --rm --network danswer-stack_default \
            --name test-runner \
            -e POSTGRES_HOST=relational_db \
            -e POSTGRES_USER=postgres \
            -e POSTGRES_PASSWORD=password \
            -e POSTGRES_DB=postgres \
            -e VESPA_HOST=index \
            -e REDIS_HOST=cache \
            -e API_SERVER_HOST=api_server \
            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
            -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
            -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
            -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
            -e TEST_WEB_HOSTNAME=test-runner \
            -e PYTEST_ADDOPTS="-n auto --dist=loadscope" \
            onyxdotapp/onyx-integration:test \
            pytest /app/tests/integration/connector_job_tests
        id: run_connector_tests

      - name: Save Docker logs
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log
          mv docker-compose.log ${{ github.workspace }}/docker-compose.log

      - name: Upload logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: docker-logs
          path: ${{ github.workspace }}/docker-compose.log

      - name: Stop Docker containers
        if: always()
        run: |
          cd deployment/docker_compose
          docker compose -f docker-compose.dev.yml -p danswer-stack down -v