Improve speed and stability of Python 3.12 tests in canary build #92961
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| name: Tests | |
| on: # yamllint disable-line rule:truthy | |
| schedule: | |
| - cron: '28 0 * * *' | |
| push: | |
| branches: ['main', 'v[0-9]+-[0-9]+-test'] | |
| pull_request: | |
| branches: ['main'] | |
| permissions: | |
| # All other permissions are set to none | |
| contents: read | |
| # Technically read access while waiting for images should be more than enough. However, | |
| # there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission | |
| # denied error when attempting to just pull private image, changing the token permission to write solves the | |
| # issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml | |
| # changes. If it is for a fork, then the token is read-only anyway. | |
| packages: write | |
| env: | |
| ANSWER: "yes" | |
| DB_RESET: "true" | |
| VERBOSE: "true" | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the | |
| # Airflow one is going to be used | |
| CONSTRAINTS_GITHUB_REPOSITORY: >- | |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && | |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} | |
| # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| USE_SUDO: "true" | |
| INCLUDE_SUCCESS_OUTPUTS: "true" | |
| INCLUDE_NOT_READY_PROVIDERS: "true" | |
| AIRFLOW_ENABLE_AIP_44: "true" | |
| MOUNT_SOURCES: "skip" | |
| concurrency: | |
| group: ci-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-info: | |
| name: "Build info" | |
| # The runs-on cannot refer to env. or secrets. context, so we have no | |
| # option but to specify a hard-coded list here. This is "safe", the | |
| # runner checks if the user is an owner or collaborator of the repo | |
| # before running the workflow. | |
| runs-on: ["ubuntu-22.04"] | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| outputs: | |
| # Version of Python used for reproducibility of the packages built | |
| # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing | |
| # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible | |
| # TODO: remove me when we switch to Python 3.9 as minimal version | |
| breeze-python-version: "3.9" | |
| image-tag: ${{ github.event.pull_request.head.sha || github.sha }} | |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} | |
| affected-providers-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} | |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} | |
| python-versions: ${{ steps.selective-checks.outputs.python-versions }} | |
| python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} | |
| all-python-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} | |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} | |
| kubernetes-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} | |
| kubernetes-combos-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} | |
| default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} | |
| postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} | |
| default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} | |
| mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} | |
| default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} | |
| default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} | |
| default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} | |
| full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} | |
| parallel-test-types-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} | |
| postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} | |
| mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} | |
| sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} | |
| skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }} | |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} | |
| run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} | |
| run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} | |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} | |
| ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} | |
| prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} | |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} | |
| mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }} | |
| needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }} | |
| needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} | |
| needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} | |
| needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} | |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} | |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} | |
| docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} | |
| skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} | |
| providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }} | |
| helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} | |
| debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} | |
| runs-on: ${{steps.selective-checks.outputs.runs-on}} | |
| is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} | |
| is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} | |
| is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} | |
| is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} | |
| is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} | |
| is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} | |
| latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} | |
| chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} | |
| has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} | |
| source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} | |
| pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} | |
| in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }} | |
| build-job-description: ${{ steps.source-run-info.outputs.build-job-description }} | |
| canary-run: ${{ steps.source-run-info.outputs.canary-run }} | |
| run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: Fetch incoming commit ${{ github.sha }} with its parent | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Get information about the Workflow" | |
| id: source-run-info | |
| run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} | |
| - name: Selective checks | |
| id: selective-checks | |
| env: | |
| PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" | |
| COMMIT_REF: "${{ github.sha }}" | |
| VERBOSE: "false" | |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} | |
| - name: env | |
| run: printenv | |
| env: | |
| PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the | |
| # tests to complete - it is run very early in the build process for "main" merges in order to refresh | |
| # cache using the current constraints. This will speed up cache refresh in cases when pyproject.toml | |
| # changes or in case of Dockerfile changes. Failure in this step is not a problem (at most it will | |
| # delay cache refresh. It does not attempt to upgrade to newer dependencies. | |
| # We only push CI cache as PROD cache usually does not gain as much from fresh cache because | |
| # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time | |
| push-early-buildx-cache-to-github-registry: | |
| name: Push Early Image Cache | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/push-image-cache.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| cache-type: "Early" | |
| include-prod-images: "false" | |
| push-latest-images: "false" | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| use-uv: "true" | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| # Check that after earlier cache push, breeze command will build quickly | |
| check-that-image-builds-quickly: | |
| timeout-minutes: 5 | |
| name: "Check that image builds quickly" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - push-early-buildx-cache-to-github-registry | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: false | |
| PLATFORM: "linux/amd64" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: ${{needs.build-info.outputs.default-python-version}} | |
| if: > | |
| needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Check that image builds quickly" | |
| run: breeze shell --max-time 120 | |
| build-ci-images: | |
| name: > | |
| ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} | |
| CI images in-workflow | |
| needs: [build-info] | |
| uses: ./.github/workflows/ci-image-build.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| do-build: ${{ needs.build-info.outputs.in-workflow-build }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| upload-constraints: "true" | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| use-uv: "true" | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| run-breeze-tests: | |
| timeout-minutes: 10 | |
| name: Breeze unit tests | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Need to fetch all history for selective checks tests | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "${{needs.build-info.outputs.default-python-version}}" | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/breeze/setup* | |
| - run: pip install --editable ./dev/breeze/ | |
| - run: python -m pytest -n auto --color=yes | |
| working-directory: ./dev/breeze/ | |
| tests-www: | |
| timeout-minutes: 10 | |
| name: React WWW tests | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| if: needs.build-info.outputs.run-www-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Setup node" | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 21 | |
| - name: "Cache eslint" | |
| uses: actions/cache@v4 | |
| with: | |
| path: 'airflow/www/node_modules' | |
| key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }} | |
| - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive | |
| - run: yarn --cwd airflow/www/ run test | |
| env: | |
| FORCE_COLOR: 2 | |
| test-openapi-client: | |
| timeout-minutes: 10 | |
| name: "Test OpenAPI client" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| if: needs.build-info.outputs.needs-api-codegen == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: "apache/airflow-client-python" | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| path: ./airflow-client-python | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Generate client with breeze" | |
| run: > | |
| breeze release-management prepare-python-client --package-format both | |
| --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python | |
| - name: "Show diff" | |
| run: git diff --color HEAD | |
| working-directory: ./airflow-client-python | |
| - name: Install hatch | |
| run: | | |
| python -m pip install --upgrade pipx | |
| pipx ensurepath | |
| pipx install hatch --force | |
| - name: Run tests | |
| run: hatch run run-coverage | |
| env: | |
| HATCH_ENV: "test" | |
| working-directory: ./clients/python | |
| - name: "Install Airflow in editable mode with fab for webserver tests" | |
| run: pip install -e ".[fab]" | |
| - name: "Install Python client" | |
| run: pip install ./dist/apache_airflow_client-*.whl | |
| - name: "Initialize Airflow DB and start webserver" | |
| run: | | |
| airflow db init | |
| # Let scheduler runs a few loops and get all DAG files from example DAGs serialized to DB | |
| airflow scheduler --num-runs 100 | |
| airflow users create --username admin --password admin --firstname Admin --lastname Admin \ | |
| --role Admin --email admin@example.org | |
| killall python || true # just in case there is a webserver running in the background | |
| nohup airflow webserver --port 8080 & | |
| echo "Started webserver" | |
| env: | |
| AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.session,airflow.api.auth.backend.basic_auth | |
| AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True" | |
| AIRFLOW__CORE__LOAD_EXAMPLES: "True" | |
| - name: "Waiting for the webserver to be available" | |
| run: | | |
| timeout 30 bash -c 'until nc -z $0 $1; do echo "sleeping"; sleep 1; done' localhost 8080 | |
| sleep 5 | |
| - name: "Run test python client" | |
| run: python ./clients/python/test_python_client.py | |
| env: | |
| FORCE_COLOR: "standard" | |
| - name: "Stop running webserver" | |
| run: killall python || true # just in case there is a webserver running in the background | |
| if: always() | |
| - name: "Upload python client packages" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: python-client-packages | |
| path: ./dist/apache_airflow_client-* | |
| retention-days: 7 | |
| if-no-files-found: error | |
| test-git-clone-on-windows: | |
| timeout-minutes: 5 | |
| name: "Test git clone on Windows" | |
| runs-on: "windows-latest" | |
| needs: [build-info] | |
| steps: | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| wait-for-ci-images: | |
| timeout-minutes: 120 | |
| name: "Wait for CI images" | |
| runs-on: ["ubuntu-22.04"] | |
| needs: [build-info, build-ci-images] | |
| if: needs.build-info.outputs.ci-image-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| id: wait-for-images | |
| run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| generate-constraints: | |
| permissions: | |
| contents: read | |
| timeout-minutes: 70 | |
| name: > | |
| Generate constraints with CI image verifying | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| if: needs.build-info.outputs.ci-image-build == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| with: | |
| python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze ci-image pull --run-in-parallel --tag-as-latest | |
| - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze ci-image verify --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: "Source constraints" | |
| shell: bash | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints-source-providers | |
| - name: "No providers constraints" | |
| shell: bash | |
| timeout-minutes: 25 | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints-no-providers | |
| - name: "Prepare chicken-eggs provider packages" | |
| # In case of provider packages which use latest dev0 version of providers, we should prepare them | |
| # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency | |
| # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 | |
| shell: bash | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --package-format wheel --version-suffix-for-pypi dev0 | |
| ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| if: needs.build-info.outputs.chicken-egg-providers != '' | |
| - name: "PyPI constraints" | |
| shell: bash | |
| timeout-minutes: 25 | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints | |
| --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}" | |
| - name: "Dependency upgrade summary" | |
| shell: bash | |
| run: | | |
| for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do | |
| echo "Summarizing Python $PYTHON_VERSION" | |
| cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true | |
| done | |
| - name: "Upload constraint artifacts" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: constraints | |
| path: ./files/constraints-*/constraints-*.txt | |
| retention-days: 7 | |
| if-no-files-found: error | |
| static-checks: | |
| timeout-minutes: 45 | |
| name: "Static checks" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| if: > | |
| needs.build-info.outputs.basic-checks-only == 'false' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| id: breeze | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| # yamllint disable-line rule:line-length | |
| key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" | |
| restore-keys: | | |
| pre-commit-${{steps.breeze.outputs.host-python-version}}- | |
| - name: "Static checks" | |
| run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment | |
| env: | |
| VERBOSE: "false" | |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} | |
| COLUMNS: "250" | |
| SKIP_GROUP_OUTPUT: "true" | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| RUFF_FORMAT: "github" | |
| upgrade-check: | |
| timeout-minutes: 45 | |
| name: "Upgrade checks" | |
| runs-on: ["ubuntu-22.04"] | |
| needs: [build-info] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| if: > | |
| needs.build-info.outputs.canary-run == 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| # Install python from scratch. No cache used. We always want to have fresh version of everything | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "${{needs.build-info.outputs.default-python-version}}" | |
| - name: "Install latest pre-commit" | |
| run: pip install pre-commit | |
| - name: "Autoupdate all pre-commits" | |
| run: pre-commit autoupdate | |
| - name: "Run automated upgrade for black" | |
| run: > | |
| pre-commit run | |
| --all-files --show-diff-on-failure --color always --verbose | |
| --hook-stage manual | |
| update-black-version | |
| if: always() | |
| - name: "Run automated upgrade for build dependencies" | |
| run: > | |
| pre-commit run | |
| --all-files --show-diff-on-failure --color always --verbose | |
| --hook-stage manual | |
| update-build-dependencies | |
| if: always() | |
| - name: "Run automated upgrade for chart dependencies" | |
| run: > | |
| pre-commit run | |
| --all-files --show-diff-on-failure --color always --verbose | |
| --hook-stage manual | |
| update-chart-dependencies | |
| if: always() | |
| # For UV we are not failing the upgrade installers check if it is updated because | |
| # it is upgraded very frequently, so we want to manually upgrade it rather than | |
| # get notified about it - until it stabilizes in 1.* version | |
| - name: "Run automated upgrade for uv (open to see if new version is updated)" | |
| run: > | |
| pre-commit run | |
| --all-files --show-diff-on-failure --color always --verbose | |
| --hook-stage manual update-installers || true | |
| if: always() | |
| env: | |
| UPGRADE_UV: "true" | |
| UPGRADE_PIP: "false" | |
| - name: "Run automated upgrade for pip" | |
| run: > | |
| pre-commit run | |
| --all-files --show-diff-on-failure --color always --verbose | |
| --hook-stage manual update-installers | |
| if: always() | |
| env: | |
| UPGRADE_UV: "false" | |
| UPGRADE_PIP: "true" | |
| mypy: | |
| name: "MyPy tests" | |
| uses: ./.github/workflows/mypy.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| mypy-folders: ${{ needs.build-info.outputs.mypy-folders }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
| needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} | |
| if: needs.build-info.outputs.latest-versions-only != 'true' | |
| # Those checks are run if no image needs to be built for checks. This is for simple changes that | |
| # Do not touch any of the python code or any of the important files that might require building | |
| # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host | |
| static-checks-basic-checks-only: | |
| timeout-minutes: 30 | |
| name: "Static checks: basic checks only" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| if: needs.build-info.outputs.basic-checks-only == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Setup python" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "${{needs.build-info.outputs.default-python-version}}" | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/breeze/setup* | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| id: breeze | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| # yamllint disable-line rule:line-length | |
| key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" | |
| restore-keys: "\ | |
| pre-commit-${{steps.breeze.outputs.host-python-version}}-\ | |
| ${{ hashFiles('.pre-commit-config.yaml') }}\n | |
| pre-commit-${{steps.breeze.outputs.host-python-version}}-" | |
| - name: Fetch incoming commit ${{ github.sha }} with its parent | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Static checks: basic checks only" | |
| run: > | |
| breeze static-checks --show-diff-on-failure --color always --initialize-environment | |
| --skip-image-upgrade-check --commit-ref "${{ github.sha }}" | |
| env: | |
| VERBOSE: "false" | |
| SKIP_BREEZE_PRE_COMMITS: "true" | |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} | |
| COLUMNS: "250" | |
| build-docs: | |
| timeout-minutes: 60 | |
| name: "Build docs" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: > | |
| needs.build-info.outputs.docs-build == 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - uses: actions/cache@v4 | |
| id: cache-doc-inventories | |
| with: | |
| path: ./docs/_inventory_cache/ | |
| key: docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| restore-keys: | | |
| docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| docs-inventory- | |
| - name: "Build docs" | |
| run: > | |
| breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --docs-only | |
| - name: "Clone airflow-site" | |
| run: > | |
| git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && | |
| echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Publish docs" | |
| run: > | |
| breeze release-management publish-docs | |
| --override-versioned --run-in-parallel | |
| ${{ needs.build-info.outputs.docs-list-as-string }} | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Generate back references for providers" | |
| run: breeze release-management add-back-references all-providers | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Generate back references for apache-airflow" | |
| run: breeze release-management add-back-references apache-airflow | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Generate back references for docker-stack" | |
| run: breeze release-management add-back-references docker-stack | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Generate back references for helm-chart" | |
| run: breeze release-management add-back-references helm-chart | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| with: | |
| aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-central-1 | |
| - name: "Upload documentation to AWS S3" | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs | |
| spellcheck-docs: | |
| timeout-minutes: 120 | |
| name: "Spellcheck docs" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: > | |
| needs.build-info.outputs.docs-build == 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - uses: actions/cache@v4 | |
| id: cache-doc-inventories | |
| with: | |
| path: ./docs/_inventory_cache/ | |
| key: docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| restore-keys: | | |
| docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| docs-inventory- | |
| - name: "Spellcheck docs" | |
| run: > | |
| breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --spellcheck-only | |
| prepare-test-provider-packages-wheel: | |
| timeout-minutes: 80 | |
| name: "Provider packages wheel (verify)" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| if: > | |
| needs.build-info.outputs.skip-provider-tests != 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| with: | |
| python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider documentation" | |
| run: > | |
| breeze release-management prepare-provider-documentation --include-not-ready-providers | |
| --non-interactive | |
| - name: "Prepare provider packages: wheel" | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --version-suffix-for-pypi dev0 --package-format wheel | |
| - name: "Prepare airflow package: wheel" | |
| run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 | |
| - name: "Verify wheel packages with twine" | |
| run: | | |
| pipx uninstall twine || true | |
| pipx install twine && twine check dist/*.whl | |
| - name: "Test providers issue generation automatically" | |
| run: > | |
| breeze release-management generate-issue-content-providers | |
| --only-available-in-dist --disable-progress | |
| - name: "Generate source constraints from CI image" | |
| shell: bash | |
| run: > | |
| breeze release-management generate-constraints | |
| --airflow-constraints-mode constraints-source-providers | |
| - name: "Install and verify all provider packages and airflow via wheel files" | |
| run: > | |
| breeze release-management verify-provider-packages | |
| --use-packages-from-dist | |
| --package-format wheel | |
| --use-airflow-version wheel | |
| --airflow-constraints-reference default | |
| --providers-constraints-location | |
| /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt | |
| env: | |
| AIRFLOW_SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| - name: "Prepare airflow package: wheel without suffix and skipping the tag check" | |
| run: > | |
| breeze release-management prepare-provider-packages --skip-tag-check --package-format wheel | |
| prepare-install-provider-packages-sdist: | |
| timeout-minutes: 80 | |
| name: "Provider packages sdist (install)" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| if: > | |
| needs.build-info.outputs.canary-run == 'true' && | |
| needs.build-info.outputs.skip-provider-tests != 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| with: | |
| python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider packages: sdist" | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --version-suffix-for-pypi dev0 --package-format sdist | |
| ${{ needs.build-info.outputs.affected-providers-list-as-string }} | |
| - name: "Prepare airflow package: sdist" | |
| run: > | |
| breeze release-management prepare-airflow-package | |
| --version-suffix-for-pypi dev0 --package-format sdist | |
| - name: "Verify sdist packages with twine" | |
| run: | | |
| pipx uninstall twine || true | |
| pipx install twine && twine check dist/*.tar.gz | |
| - name: "Generate source constraints from CI image" | |
| shell: bash | |
| run: > | |
| breeze release-management generate-constraints | |
| --airflow-constraints-mode constraints-source-providers | |
| - name: "Install all provider packages and airflow via sdist files" | |
| run: > | |
| breeze release-management install-provider-packages | |
| --use-packages-from-dist | |
| --package-format sdist | |
| --use-airflow-version sdist | |
| --airflow-constraints-reference default | |
| --providers-constraints-location | |
| /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt | |
| --run-in-parallel | |
| if: needs.build-info.outputs.affected-providers-list-as-string == '' | |
| - name: "Install affected provider packages and airflow via sdist files" | |
| run: > | |
| breeze release-management install-provider-packages | |
| --use-packages-from-dist | |
| --package-format sdist | |
| --use-airflow-version sdist | |
| --airflow-constraints-reference default | |
| --providers-constraints-location | |
| /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt | |
| --run-in-parallel | |
| if: needs.build-info.outputs.affected-providers-list-as-string != '' | |
| providers-compatibility-checks: | |
| timeout-minutes: 80 | |
| name: > | |
| Compat ${{matrix.airflow-version}}:P${{matrix.python-version}} provider check | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{fromJson(needs.build-info.outputs.providers-compatibility-checks)}} | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| if: needs.build-info.outputs.skip-provider-tests != 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| with: | |
| python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider packages: wheel" | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --package-format wheel | |
| - name: > | |
| Remove incompatible Airflow | |
| ${{matrix.airflow-version}}:Python ${{matrix.python-version}} provider packages | |
| run: | | |
| rm -vf ${{ matrix.remove-providers }} | |
| working-directory: ./dist | |
| if: matrix.remove-providers != '' | |
| - name: "Download airflow package: wheel" | |
| run: | | |
| pip download "apache-airflow==${{matrix.airflow-version}}" -d dist --no-deps | |
| - name: > | |
| Install and verify all provider packages and airflow on | |
| Airflow ${{matrix.airflow-version}}:Python ${{matrix.python-version}} | |
| run: > | |
| breeze release-management verify-provider-packages | |
| --use-packages-from-dist | |
| --package-format wheel | |
| --use-airflow-version wheel | |
| --airflow-constraints-reference constraints-${{matrix.airflow-version}} | |
| --providers-skip-constraints | |
| --install-airflow-with-constraints | |
| test-airflow-release-commands: | |
| timeout-minutes: 80 | |
| name: "Test Airflow release commands" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: > | |
| needs.build-info.outputs.is-self-hosted-runner == 'true' | |
| && needs.build-info.outputs.is-airflow-runner == 'false' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Install required packages" | |
| run: python -m pip install --editable ./dev/breeze/ | |
| - name: "Check Airflow create minor branch command" | |
| run: breeze release-management create-minor-branch --version-branch 2-4 -a y | |
| - name: "Check Airflow RC process command" | |
| run: | | |
| breeze release-management start-rc-process --version 2.4.3rc1 --previous-version 2.4.2 -a y | |
| - name: "Check Airflow release process command" | |
| run: | | |
| breeze release-management start-release --release-candidate 2.4.3rc1 --previous-release 2.4.2 -a y | |
| tests-helm: | |
| name: "Helm tests" | |
| uses: ./.github/workflows/helm-tests.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
| if: > | |
| needs.build-info.outputs.needs-helm-tests == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| tests-postgres: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:Postgres${{matrix.backend-version}},Py${{matrix.python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" | |
| backend-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}" | |
| exclude: "${{fromJson(needs.build-info.outputs.postgres-exclude)}}" | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| BACKEND_VERSION: "${{matrix.backend-version}}" | |
| JOB_ID: "postgres-${{matrix.backend-version}}-${{matrix.python-version}}" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: "Tests ARM Pytest collection: ${{matrix.python-version}}" | |
| run: breeze testing db-tests --collect-only --remove-arm-packages | |
| if: matrix.backend-version == needs.build-info.outputs.default-postgres-version | |
| - name: > | |
| Post Tests success: Postgres" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Postgres" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-min-sqlalchemy: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:MinSQLAlchemy${{needs.build-info.outputs.default-postgres-version}}, | |
| Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| DOWNGRADE_SQLALCHEMY: "true" | |
| JOB_ID: > | |
| min-sqlalchemy-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:MinSQLAlchemy" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:MinSQLAlchemy" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-boto: | |
| timeout-minutes: 130 | |
| name: > | |
| LatestBoto-Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| UPGRADE_BOTO: "true" | |
| JOB_ID: > | |
| boto-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing tests --run-in-parallel | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:LatestBoto" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:LatestBoto" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-pydantic: | |
| timeout-minutes: 130 | |
| name: > | |
| Pydantic-${{ matrix.pydantic }}-Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| strategy: | |
| matrix: | |
| pydantic: ["v1", "none"] | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| PYDANTIC: ${{ matrix.pydantic }} | |
| JOB_ID: > | |
| pydantic-${{ matrix.pydantic }}-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing tests --run-in-parallel | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:NoPydantic" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:NoPydantic" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-pendulum-2: | |
| timeout-minutes: 130 | |
| name: > | |
| Pendulum2-Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| DOWNGRADE_PENDULUM: "true" | |
| JOB_ID: > | |
| pendulum-2-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: > | |
| needs.build-info.outputs.run-tests == 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing tests --run-in-parallel | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Pendulum2" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Pendulum2" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-in-progress-features-disabled: | |
| timeout-minutes: 130 | |
| name: > | |
| InProgressDisabled-Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| AIRFLOW_ENABLE_AIP_44: "false" | |
| JOB_ID: > | |
| in-progress-disabled-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing tests --run-in-parallel | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-mysql: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:MySQL${{matrix.backend-version}}, Py${{matrix.python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" | |
| backend-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}" | |
| exclude: "${{fromJson(needs.build-info.outputs.mysql-exclude)}}" | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "mysql" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| BACKEND_VERSION: "${{matrix.backend-version}}" | |
| JOB_ID: "mysql-${{matrix.backend-version}}-${{matrix.python-version}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: MySQL" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: MySQL" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-sqlite: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} | |
| exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }} | |
| fail-fast: false | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "sqlite" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| BACKEND_VERSION: "" | |
| JOB_ID: "sqlite-${{matrix.python-version}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: Sqlite" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Sqlite" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-integration-postgres: | |
| timeout-minutes: 130 | |
| name: Integration Tests Postgres | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| JOB_ID: "integration-postgres" | |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Integration Tests Postgres: cassandra" | |
| run: | | |
| breeze testing integration-tests --integration cassandra | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: mongo" | |
| run: | | |
| breeze testing integration-tests --integration mongo | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: pinot" | |
| run: | | |
| breeze testing integration-tests --integration pinot | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: celery" | |
| run: | | |
| breeze testing integration-tests --integration celery | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: trino, kerberos" | |
| run: | | |
| breeze testing integration-tests --integration trino --integration kerberos | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: Kafka" | |
| run: | | |
| breeze testing integration-tests --integration kafka | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: Qdrant" | |
| run: breeze testing integration-tests --integration qdrant | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Integration Tests Postgres: all-testable" | |
| run: breeze testing integration-tests --integration all-testable | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests success: Integration Postgres" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Integration Postgres" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-integration-mysql: | |
| timeout-minutes: 130 | |
| name: Integration Tests MySQL | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "mysql" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}" | |
| JOB_ID: "integration-mysql" | |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Integration Tests MySQL: all-testable" | |
| run: breeze testing integration-tests --integration all-testable | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests success: Integration MySQL" | |
| uses: ./.github/actions/post_tests_success | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests failure: Integration MySQL" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-quarantined: | |
| timeout-minutes: 60 | |
| name: "Quarantined tests" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| continue-on-error: true | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| TEST_TYPE: "All-Quarantined" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.is-airflow-runner == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: postgres:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "postgres" | |
| BACKEND_VERSION: ${{needs.build-info.outputs.default-postgres-version}} | |
| - name: > | |
| Tests: mysql:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "mysql" | |
| BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}} | |
| - name: > | |
| Tests: sqlite:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "sqlite" | |
| BACKEND_VERSION: "" | |
| - name: > | |
| Post Tests success: Quarantined" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Quarantined" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-no-db: | |
| timeout-minutes: 60 | |
| name: > | |
| Non-DB: Py${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| JOB_ID: "no-db-${{matrix.python-version}}" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Tests: ${{matrix.python-version}}:Non-DB" | |
| run: > | |
| breeze testing non-db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: "Post Tests success: Non-DB" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: "Post Tests failure: Non-DB" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-no-db-pendulum-2: | |
| timeout-minutes: 60 | |
| name: > | |
| Non-DB: Pendulum2, Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| JOB_ID: "no-db-pendulum-2-${{needs.build-info.outputs.default-python-version}}" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| DOWNGRADE_PENDULUM: "true" | |
| if: > | |
| needs.build-info.outputs.run-tests == 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Tests: ${{matrix.python-version}}:Non-DB-Pendulum2" | |
| run: > | |
| breeze testing non-db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: "Post Tests success: Non-DB-Pendulum2" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: "Post Tests failure: Non-DB-Pendulum2" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| summarize-warnings: | |
| timeout-minutes: 15 | |
| name: "Summarize warnings" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - tests-postgres | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-quarantined | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| - tests-no-db | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Download all artifacts from the current build" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ./artifacts | |
| - name: "Summarize all warnings" | |
| run: | | |
| ls -R ./artifacts/ | |
| cat ./artifacts/test-warnings*/* | sort | uniq | |
| echo | |
| echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l) | |
| build-prod-images: | |
| name: > | |
| ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} | |
| PROD images in-workflow | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/prod-image-build.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| build-type: "Regular" | |
| do-build: ${{ needs.build-info.outputs.in-workflow-build }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| push-image: "true" | |
| use-uv: "true" | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| prod-image-extra-checks-main: | |
| name: PROD image extra checks (main) | |
| # Here we just need to wait for CI images: we build provider packages and can use source constraints | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/prod-image-extra-checks.yml | |
| with: | |
| python-versions: "[ '${{ needs.build-info.outputs.default-python-version }}' ]" | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: needs.build-info.outputs.default-branch == 'main' && needs.build-info.outputs.canary-run == 'true' | |
| prod-image-extra-checks-release-branch: | |
| name: PROD image extra checks (release) | |
| # Here we need to wait for generate-constraints to complete because we have to use PyPI constraints | |
| needs: [build-info, generate-constraints] | |
| uses: ./.github/workflows/prod-image-extra-checks.yml | |
| with: | |
| python-versions: "[ '${{ needs.build-info.outputs.default-python-version }}' ]" | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: needs.build-info.outputs.default-branch != 'main' && needs.build-info.outputs.canary-run == 'true' | |
| wait-for-prod-images: | |
| timeout-minutes: 80 | |
| name: "Wait for PROD images" | |
| runs-on: ["ubuntu-22.04"] | |
| needs: [build-info, wait-for-ci-images, build-prod-images] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| # Force more parallelism for pull on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| # We wait for the images to be available either from "build-images.yml' run as pull_request_target | |
| # or from build-prod-images (or build-prod-images-release-branch) above. | |
| # We are utilising single job to wait for all images because this job merely waits | |
| # For the images to be available. | |
| run: breeze prod-image pull --wait-for-image --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| test-examples-of-prod-image-building: | |
| timeout-minutes: 60 | |
| name: "Test examples of production image building" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-prod-images] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: > | |
| Pull PROD image | |
| "${{needs.build-info.outputs.default-python-version}}":${{ needs.build-info.outputs.image-tag }} | |
| run: breeze prod-image pull --tag-as-latest | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| - name: "Setup python" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{needs.build-info.outputs.default-python-version}} | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/requirements.txt | |
| - name: "Test examples of PROD image building" | |
| # yamllint disable-line rule:line-length | |
| run: > | |
| cd ./docker_tests && | |
| python -m pip install -r requirements.txt && | |
| TEST_IMAGE="ghcr.io/${{env.GITHUB_REPOSITORY}}/${{needs.build-info.outputs.default-branch}}/prod/python${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| test-docker-compose-quick-start: | |
| timeout-minutes: 60 | |
| name: "Docker-compose quick start with PROD image verifying" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-prod-images] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| - name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze prod-image verify --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: "Test docker-compose quick start" | |
| run: breeze testing docker-compose-tests | |
| tests-kubernetes: | |
| timeout-minutes: 240 | |
| name: "\ | |
| K8S System:${{matrix.executor}} - ${{matrix.use-standard-naming}} - \ | |
| ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-prod-images] | |
| strategy: | |
| matrix: | |
| executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] | |
| use-standard-naming: [true, false] | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: > | |
| ( needs.build-info.outputs.run-kubernetes-tests == 'true' || | |
| needs.build-info.outputs.needs-helm-tests == 'true') | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| id: breeze | |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| - name: "Cache bin folder with tools for kubernetes testing" | |
| uses: actions/cache@v4 | |
| with: | |
| path: ".build/.k8s-env" | |
| key: "\ | |
| k8s-env-${{steps.breeze.outputs.host-python-version}}-\ | |
| ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','pyproject.toml') }}" | |
| - name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| run: breeze k8s run-complete-tests --run-in-parallel --upgrade | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} | |
| EXECUTOR: ${{matrix.executor}} | |
| USE_STANDARD_NAMING: ${{matrix.use-standard-naming}} | |
| VERBOSE: "false" | |
| - name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| uses: actions/upload-artifact@v4 | |
| if: failure() || cancelled() | |
| with: | |
| name: kind-logs-${{matrix.executor}}-${{matrix.use-standard-naming}} | |
| path: /tmp/kind_logs_* | |
| retention-days: 7 | |
| - name: > | |
| Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| uses: actions/upload-artifact@v4 | |
| if: failure() || cancelled() | |
| with: | |
| name: k8s-test-resources-${{matrix.executor}}-${{matrix.use-standard-naming}} | |
| path: /tmp/k8s_test_resources_* | |
| retention-days: 7 | |
| - name: "Delete clusters just in case they are left" | |
| run: breeze k8s delete-cluster --all | |
| if: always() | |
| update-constraints: | |
| permissions: | |
| contents: write | |
| packages: write | |
| timeout-minutes: 80 | |
| name: "Update constraints" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - build-docs | |
| - spellcheck-docs | |
| - wait-for-ci-images | |
| - wait-for-prod-images | |
| - static-checks | |
| - mypy | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-postgres | |
| - tests-no-db | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| - generate-constraints | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| # Needed to perform push action | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Set constraints branch name" | |
| id: constraints-branch | |
| run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} | |
| - name: Checkout ${{ steps.constraints-branch.outputs.branch }} | |
| uses: actions/checkout@v4 | |
| with: | |
| path: "constraints" | |
| ref: ${{ steps.constraints-branch.outputs.branch }} | |
| persist-credentials: true | |
| fetch-depth: 0 | |
| - name: "Download constraints from the constraints generated by build CI image" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: constraints | |
| path: ./files | |
| - name: "Diff in constraints for ${{needs.build-info.outputs.python-versions}}" | |
| run: ./scripts/ci/constraints/ci_diff_constraints.sh | |
| # only commit and push constraints in canary runs (main) | |
| - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}" | |
| run: ./scripts/ci/constraints/ci_commit_constraints.sh | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Push changes" | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| working-directory: "constraints" | |
| run: | |
| git push | |
| # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build | |
| # is executed as result of direct push to "main" or one of the "vX-Y-test" branches | |
| # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry | |
| # It will automatically check if a new python image was released and will pull the latest one if needed | |
| push-buildx-cache-to-github-registry: | |
| name: Push Regular Image Cache | |
| needs: [build-info, update-constraints] | |
| uses: ./.github/workflows/push-image-cache.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| cache-type: "Regular" | |
| include-prod-images: "true" | |
| push-latest-images: "true" | |
| use-uv: "true" | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| # This is only a check if ARM images are successfully building when committer runs PR from | |
| # Apache repository. This is needed in case you want to fix failing cache job in "canary" run | |
| # There is no point in running this one in "canary" run, because the above step is doing the | |
| # same build anyway. | |
| build-ci-arm-images: | |
| name: Build CI ARM images (in-workflow) | |
| needs: | |
| - build-info | |
| - build-docs | |
| - spellcheck-docs | |
| - static-checks | |
| - mypy | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-postgres | |
| - tests-no-db | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| uses: ./.github/workflows/ci-image-build.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| platform: "arm64" | |
| runs-on: ${{needs.build-info.outputs.runs-on}} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| use-uv: "true" | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.canary-run != 'true' |