Parallelize Integration Tests #1804
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Integration Tests v2 | |
| concurrency: | |
| group: Run-Integration-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }} | |
| cancel-in-progress: true | |
| on: | |
| merge_group: | |
| pull_request: | |
| branches: | |
| - main | |
| - "release/**" | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} | |
| CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} | |
| CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} | |
| jobs: | |
| build-images: | |
| # See https://runs-on.com/runners/linux/ | |
| runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| # tag every docker image with "test" so that we can spin up the correct set | |
| # of images during testing | |
| # We don't need to build the Web Docker image since it's not yet used | |
| # in the integration tests. We have a separate action to verify that it builds | |
| # successfully. | |
| - name: Pull Web Docker image | |
| run: | | |
| docker pull onyxdotapp/onyx-web-server:latest | |
| docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test | |
| # we use the runs-on cache for docker builds | |
| # in conjunction with runs-on runners, it has better speed and unlimited caching | |
| # https://runs-on.com/caching/s3-cache-for-github-actions/ | |
| # https://runs-on.com/caching/docker/ | |
| # https://github.com/moby/buildkit#s3-cache-experimental | |
| # images are built and run locally for testing purposes. Not pushed. | |
| - name: Build Backend Docker image | |
| uses: ./.github/actions/custom-build-and-push | |
| with: | |
| context: ./backend | |
| file: ./backend/Dockerfile | |
| platforms: linux/amd64 | |
| tags: onyxdotapp/onyx-backend:test | |
| push: false | |
| load: true | |
| cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} | |
| cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max | |
| - name: Build Model Server Docker image | |
| uses: ./.github/actions/custom-build-and-push | |
| with: | |
| context: ./backend | |
| file: ./backend/Dockerfile.model_server | |
| platforms: linux/amd64 | |
| tags: onyxdotapp/onyx-model-server:test | |
| push: false | |
| load: true | |
| cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} | |
| cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max | |
| - name: Build integration test Docker image | |
| uses: ./.github/actions/custom-build-and-push | |
| with: | |
| context: ./backend | |
| file: ./backend/tests/integration/Dockerfile | |
| platforms: linux/amd64 | |
| tags: onyxdotapp/onyx-integration:test | |
| push: false | |
| load: true | |
| cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} | |
| cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max | |
| - name: Save and compress Docker images | |
| run: | | |
| # Save and compress images in parallel | |
| docker save onyxdotapp/onyx-web-server:test | gzip > web-server.tar.gz & | |
| docker save onyxdotapp/onyx-backend:test | gzip > backend.tar.gz & | |
| docker save onyxdotapp/onyx-model-server:test | gzip > model-server.tar.gz & | |
| docker save onyxdotapp/onyx-integration:test | gzip > integration.tar.gz & | |
| wait | |
| - name: Upload Docker images | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-images | |
| compression-level: 9 # Maximum compression | |
| retention-days: 1 # Minimize storage | |
| path: | | |
| web-server.tar.gz | |
| backend.tar.gz | |
| model-server.tar.gz | |
| integration.tar.gz | |
| multi-tenant-tests: | |
| needs: build-images | |
| runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download Docker images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: docker-images | |
| - name: Load Docker images | |
| run: | | |
| # Decompress and load images in parallel | |
| gunzip -c web-server.tar.gz | docker load & | |
| gunzip -c backend.tar.gz | docker load & | |
| gunzip -c model-server.tar.gz | docker load & | |
| gunzip -c integration.tar.gz | docker load & | |
| wait | |
| - name: Start Docker containers for multi-tenant tests | |
| run: | | |
| cd deployment/docker_compose | |
| ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \ | |
| MULTI_TENANT=true \ | |
| AUTH_TYPE=basic \ | |
| REQUIRE_EMAIL_VERIFICATION=false \ | |
| DISABLE_TELEMETRY=true \ | |
| IMAGE_TAG=test \ | |
| docker compose -f docker-compose.dev.yml -p danswer-stack up -d | |
| id: start_docker_multi_tenant | |
| - name: Wait for services to be ready | |
| uses: ./.github/actions/wait-for-services | |
| - name: Run Multi-Tenant Integration Tests | |
| run: | | |
| echo "Running multi-tenant integration tests..." | |
| docker run --rm --network danswer-stack_default \ | |
| --name test-runner \ | |
| -e POSTGRES_HOST=relational_db \ | |
| -e POSTGRES_USER=postgres \ | |
| -e POSTGRES_PASSWORD=password \ | |
| -e POSTGRES_DB=postgres \ | |
| -e VESPA_HOST=index \ | |
| -e REDIS_HOST=cache \ | |
| -e API_SERVER_HOST=api_server \ | |
| -e OPENAI_API_KEY=${OPENAI_API_KEY} \ | |
| -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \ | |
| -e TEST_WEB_HOSTNAME=test-runner \ | |
| -e AUTH_TYPE=cloud \ | |
| -e MULTI_TENANT=true \ | |
| -e PYTEST_ADDOPTS="-n auto --dist=loadscope" \ | |
| onyxdotapp/onyx-integration:test \ | |
| pytest /app/tests/integration/multitenant_tests | |
| id: run_multitenant_tests | |
| - name: Save Docker logs | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log | |
| mv docker-compose.log ${{ github.workspace }}/docker-compose.log | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-logs | |
| path: ${{ github.workspace }}/docker-compose.log | |
| - name: Stop Docker containers | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p danswer-stack down -v | |
| standard-tests: | |
| needs: build-images | |
| runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download Docker images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: docker-images | |
| - name: Load Docker images | |
| run: | | |
| # Decompress and load images in parallel | |
| gunzip -c web-server.tar.gz | docker load & | |
| gunzip -c backend.tar.gz | docker load & | |
| gunzip -c model-server.tar.gz | docker load & | |
| gunzip -c integration.tar.gz | docker load & | |
| wait | |
| - name: Start Docker containers | |
| run: | | |
| cd deployment/docker_compose | |
| ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \ | |
| AUTH_TYPE=basic \ | |
| REQUIRE_EMAIL_VERIFICATION=false \ | |
| DISABLE_TELEMETRY=true \ | |
| IMAGE_TAG=test \ | |
| docker compose -f docker-compose.dev.yml -p danswer-stack up -d | |
| id: start_docker | |
| - name: Wait for services to be ready | |
| uses: ./.github/actions/wait-for-services | |
| - name: Run Standard Integration Tests | |
| run: | | |
| echo "Running standard integration tests..." | |
| docker run --rm --network danswer-stack_default \ | |
| --name test-runner \ | |
| -e POSTGRES_HOST=relational_db \ | |
| -e POSTGRES_USER=postgres \ | |
| -e POSTGRES_PASSWORD=password \ | |
| -e POSTGRES_DB=postgres \ | |
| -e VESPA_HOST=index \ | |
| -e REDIS_HOST=cache \ | |
| -e API_SERVER_HOST=api_server \ | |
| -e OPENAI_API_KEY=${OPENAI_API_KEY} \ | |
| -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \ | |
| -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \ | |
| -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \ | |
| -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \ | |
| -e TEST_WEB_HOSTNAME=test-runner \ | |
| -e PYTEST_ADDOPTS="-n auto --dist=loadscope" \ | |
| onyxdotapp/onyx-integration:test \ | |
| pytest /app/tests/integration/tests | |
| id: run_standard_tests | |
| - name: Save Docker logs | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log | |
| mv docker-compose.log ${{ github.workspace }}/docker-compose.log | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-logs | |
| path: ${{ github.workspace }}/docker-compose.log | |
| - name: Stop Docker containers | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p danswer-stack down -v | |
| connector-tests: | |
| needs: build-images | |
| runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download Docker images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: docker-images | |
| - name: Load Docker images | |
| run: | | |
| # Decompress and load images in parallel | |
| gunzip -c web-server.tar.gz | docker load & | |
| gunzip -c backend.tar.gz | docker load & | |
| gunzip -c model-server.tar.gz | docker load & | |
| gunzip -c integration.tar.gz | docker load & | |
| wait | |
| - name: Start Docker containers | |
| run: | | |
| cd deployment/docker_compose | |
| ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \ | |
| AUTH_TYPE=basic \ | |
| REQUIRE_EMAIL_VERIFICATION=false \ | |
| DISABLE_TELEMETRY=true \ | |
| IMAGE_TAG=test \ | |
| docker compose -f docker-compose.dev.yml -p danswer-stack up -d | |
| id: start_docker | |
| - name: Wait for services to be ready | |
| uses: ./.github/actions/wait-for-services | |
| - name: Run Connector Integration Tests | |
| run: | | |
| echo "Running connector integration tests..." | |
| docker run --rm --network danswer-stack_default \ | |
| --name test-runner \ | |
| -e POSTGRES_HOST=relational_db \ | |
| -e POSTGRES_USER=postgres \ | |
| -e POSTGRES_PASSWORD=password \ | |
| -e POSTGRES_DB=postgres \ | |
| -e VESPA_HOST=index \ | |
| -e REDIS_HOST=cache \ | |
| -e API_SERVER_HOST=api_server \ | |
| -e OPENAI_API_KEY=${OPENAI_API_KEY} \ | |
| -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \ | |
| -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \ | |
| -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \ | |
| -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \ | |
| -e TEST_WEB_HOSTNAME=test-runner \ | |
| -e PYTEST_ADDOPTS="-n auto --dist=loadscope" \ | |
| onyxdotapp/onyx-integration:test \ | |
| pytest /app/tests/integration/connector_job_tests | |
| id: run_connector_tests | |
| - name: Save Docker logs | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log | |
| mv docker-compose.log ${{ github.workspace }}/docker-compose.log | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-logs | |
| path: ${{ github.workspace }}/docker-compose.log | |
| - name: Stop Docker containers | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p danswer-stack down -v |