Merge branch 'main' into lorin/databricks-sql-operator-on_kill-equiva…

…lent
apache · Nov 12, 2024 · 52dd8b9 · 52dd8b9
2 parents 5b175f2 + f924ecb
commit 52dd8b9
Show file tree

Hide file tree

Showing 886 changed files with 42,676 additions and 19,739 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -128,8 +128,8 @@ github:
     - sunank200
     - vatsrahul1001
     - cmarteepants
-    - gopidesupavan
     - bugraoz93
+    - briana-okyere
 
 notifications:
   jobs: jobs@airflow.apache.org

diff --git a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml
@@ -25,7 +25,7 @@ body:
         the latest release or main to see if the issue is fixed before reporting it.
       multiple: false
       options:
-        - "2.10.2"
+        - "2.10.3"
         - "main (development)"
         - "Other Airflow 2 version (please specify below)"
     validations:

diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Install pre-commit'
+description: 'Installs pre-commit and related packages'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    default: 3.9
+  uv-version:
+    description: 'uv version to use'
+    default: 0.4.30
+  pre-commit-version:
+    description: 'pre-commit version to use'
+    default: 4.0.1
+  pre-commit-uv-version:
+    description: 'pre-commit-uv version to use'
+    default: 4.1.4
+runs:
+  using: "composite"
+  steps:
+    - name: Install pre-commit, uv, and pre-commit-uv
+      shell: bash
+      run: >
+        pip install
+        pre-commit==${{inputs.pre-commit-version}}
+        uv==${{inputs.uv-version}}
+        pre-commit-uv==${{inputs.pre-commit-uv-version}}
+    - name: Cache pre-commit envs
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pre-commit
+        key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
+        restore-keys: |
+          pre-commit-${{inputs.python-version}}-
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
@@ -626,7 +626,7 @@ labelPRBasedOnFilePath:
 
   area:Scheduler:
     - airflow/jobs/**/*
-    - airflow/task/task_runner/**/*
+    - airflow/task/standard_task_runner.py
     - airflow/dag_processing/**/*
     - docs/apache-airflow/administration-and-deployment/scheduler.rst
     - tests/jobs/**/*

diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
@@ -187,25 +187,21 @@ jobs:
         working-directory: ./airflow-client-python
       - name: Install hatch
         run: |
-          python -m pip install --upgrade pipx
-          pipx ensurepath
-          pipx install hatch --force
+          python -m pip install --upgrade uv
+          uv tool install hatch
       - name: Run tests
         run: hatch run run-coverage
         env:
           HATCH_ENV: "test"
         working-directory: ./clients/python
-      - name: "Prepare FAB+standard provider packages: wheel"
-        run: >
-            breeze release-management prepare-provider-packages fab standard \
-              --package-format wheel --skip-tag-check
-      - name: "Install Airflow with fab for webserver tests"
-        run: pip install . dist/apache_airflow_providers_fab-*.whl
-      - name: "Prepare Standard provider packages: wheel"
-        run: >
-          breeze release-management prepare-provider-packages standard --package-format wheel --skip-tag-check
-      - name: "Install Airflow with standard provider for webserver tests"
-        run: pip install . dist/apache_airflow_providers_standard-*.whl
+      - name: "Install source version of required packages"
+        run: |
+            breeze release-management prepare-provider-packages fab standard common.sql --package-format  \
+            wheel --skip-tag-check --version-suffix-for-pypi dev0
+            pip install . dist/apache_airflow_providers_fab-*.whl \
+            dist/apache_airflow_providers_standard-*.whl dist/apache_airflow_providers_common_sql-*.whl
+            breeze release-management prepare-task-sdk-package --package-format wheel
+            pip install ./dist/apache_airflow_task_sdk-*.whl
       - name: "Install Python client"
         run: pip install ./dist/apache_airflow_client-*.whl
       - name: "Initialize Airflow DB and start webserver"
@@ -283,16 +279,11 @@ jobs:
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         id: breeze
-      - name: Cache pre-commit envs
-        uses: actions/cache@v4
+      - name: "Install pre-commit"
+        uses: ./.github/actions/install-pre-commit
+        id: pre-commit
         with:
-          path: ~/.cache/pre-commit
-          # yamllint disable-line rule:line-length
-          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
-          restore-keys: "\
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-\
-            ${{ hashFiles('.pre-commit-config.yaml') }}\n
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-"
+          python-version: ${{steps.breeze.outputs.host-python-version}}
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -204,7 +204,7 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -249,7 +249,7 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: linux/amd64
       python-versions: ${{ needs.build-info.outputs.python-versions }}

diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml
@@ -102,8 +102,8 @@ jobs:
         run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
       - name: "Verify wheel packages with twine"
         run: |
-          pipx uninstall twine || true
-          pipx install twine && twine check dist/*.whl
+          uv tool uninstall twine || true
+          uv tool install twine && twine check dist/*.whl
       - name: "Test providers issue generation automatically"
         run: >
           breeze release-management generate-issue-content-providers
@@ -170,8 +170,8 @@ jobs:
           --version-suffix-for-pypi dev0 --package-format sdist
       - name: "Verify sdist packages with twine"
         run: |
-          pipx uninstall twine || true
-          pipx install twine && twine check dist/*.tar.gz
+          uv tool uninstall twine || true
+          uv tool install twine && twine check dist/*.tar.gz
       - name: "Generate source constraints from CI image"
         shell: bash
         run: >

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -127,6 +127,7 @@ jobs:
       latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
       chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
       has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
+      only-new-ui-files: ${{ steps.selective-checks.outputs.only-new-ui-files }}
       source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
       pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
       in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
@@ -206,7 +207,7 @@ jobs:
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
-      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
@@ -272,14 +273,16 @@ jobs:
       latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
 
 
   generate-constraints:
     name: "Generate constraints"
     needs: [build-info, wait-for-ci-images]
     uses: ./.github/workflows/generate-constraints.yml
-    if: needs.build-info.outputs.ci-image-build == 'true'
+    if: >
+      needs.build-info.outputs.ci-image-build == 'true' &&
+      needs.build-info.outputs.only-new-ui-files != 'true'
     with:
       runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
@@ -314,6 +317,7 @@ jobs:
       ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+      docs-build: ${{ needs.build-info.outputs.docs-build }}
 
   providers:
     name: "Provider checks"
@@ -380,7 +384,7 @@ jobs:
       run-migration-tests: "true"
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
 
   tests-mysql:
     name: "MySQL tests"
@@ -405,7 +409,7 @@ jobs:
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       run-migration-tests: "true"
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
 
   tests-sqlite:
     name: "Sqlite tests"
@@ -432,7 +436,7 @@ jobs:
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       run-migration-tests: "true"
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
 
   tests-non-db:
     name: "Non-DB tests"
@@ -458,7 +462,7 @@ jobs:
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
 
   tests-special:
     name: "Special tests"
@@ -558,7 +562,7 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       push-image: "true"
-      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
@@ -641,6 +645,7 @@ jobs:
       kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }}
       kubernetes-combos-list-as-string: ${{ needs.build-info.outputs.kubernetes-combos-list-as-string }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
     if: >
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
@@ -662,7 +667,8 @@ jobs:
       run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }}
     if: >
       ( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
-      needs.build-info.outputs.run-tests == 'true')
+      needs.build-info.outputs.run-tests == 'true' &&
+      needs.build-info.outputs.only-new-ui-files != 'true')
 
   finalize-tests:
     name: Finalize tests

diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml
@@ -149,7 +149,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
+      use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -59,6 +59,7 @@ on:  # yamllint disable-line rule:truthy
 jobs:
   tests-integration:
     timeout-minutes: 130
+    if: inputs.testable-integrations != '[]'
     name: "Integration Tests: ${{ matrix.integration }}"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     strategy:

diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
@@ -44,6 +44,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to include success outputs"
         required: true
         type: string
+      use-uv:
+        description: "Whether to use uv"
+        required: true
+        type: string
       debug-resources:
         description: "Whether to debug resources"
         required: true
@@ -96,6 +100,9 @@ jobs:
           key: "\
             k8s-env-${{ steps.breeze.outputs.host-python-version }}-\
             ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','hatch_build.py') }}"
+      - name: "Switch breeze to use uv"
+        run: breeze setup config --use-uv
+        if: inputs.use-uv == 'true'
       - name: Run complete K8S tests ${{ inputs.kubernetes-combos-list-as-string }}
         run: breeze k8s run-complete-tests --run-in-parallel --upgrade --no-copy-local-sources
         env:

diff --git a/.github/workflows/news-fragment.yml b/.github/workflows/news-fragment.yml
@@ -38,7 +38,8 @@ jobs:
 
       - name: Check news fragment
         run: >
-          pipx run towncrier check
+          python -m pip install --upgrade uv &&
+          uv tool run towncrier check
           --dir .
           --config newsfragments/config.toml
           --compare-with origin/${{ github.base_ref }}

diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
@@ -181,6 +181,11 @@ jobs:
         run: >
           breeze release-management prepare-airflow-package --package-format wheel
         if: inputs.do-build  == 'true' && inputs.upload-package-artifact == 'true'
+      - name: "Prepare task-sdk package"
+        shell: bash
+        run: >
+          breeze release-management prepare-task-sdk-package --package-format wheel
+        if: inputs.do-build  == 'true' && inputs.upload-package-artifact == 'true'
       - name: "Upload prepared packages as artifacts"
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml
@@ -85,6 +85,7 @@ jobs:
       "kaxil",
       "pierrejeambrun",
       "potiuk",
+      "utkarsharma2"
       ]'), github.event.sender.login)
     steps:
       - name: "Cleanup repo"