Merge branch 'main' into lorin/databricks-sql-operator-on_kill-equiva…

…lent

.asf.yaml

@@ -122,7 +122,6 @@ github:
     # Max 10 collaborators allowed
     # https://github.com/apache/infrastructure-asfyaml/blob/main/README.md#assigning-the-github-triage-role-to-external-collaborators
     - aritra24
-    - dirrao
     - omkar-foss
     - rawwar
     - nathadfield

.github/CODEOWNERS

@@ -11,7 +11,7 @@
 
 # Kubernetes
 /airflow/kubernetes/ @dstandish @jedcunningham
-/airflow/providers/cncf/kubernetes/executors/ @dstandish @jedcunningham
+/providers/src/airflow/providers/cncf/kubernetes/executors/ @dstandish @jedcunningham
 
 # Helm Chart
 /chart/ @dstandish @jedcunningham @hussein-awala
@@ -61,27 +61,28 @@
 /airflow/secrets  @dstandish @potiuk @ashb
 
 # Providers
-/airflow/providers/amazon/ @eladkal @o-nikolas
-/airflow/providers/celery/ @hussein-awala
-/airflow/providers/cncf/kubernetes @jedcunningham @hussein-awala
-/airflow/providers/common/sql/ @eladkal
-/airflow/providers/dbt/cloud/ @josh-fell
-/airflow/providers/edge @jscheffl
-/airflow/providers/hashicorp/ @hussein-awala
-/airflow/providers/openlineage/ @mobuchowski
-/airflow/providers/slack/ @eladkal
-/airflow/providers/smtp/ @hussein-awala
-/airflow/providers/snowflake/ @potiuk @mik-laj
-/airflow/providers/tabular/ @Fokko
+/providers/src/airflow/providers/amazon/ @eladkal @o-nikolas
+/providers/src/airflow/providers/celery/ @hussein-awala
+/providers/src/airflow/providers/cncf/kubernetes @jedcunningham @hussein-awala
+/providers/src/airflow/providers/common/sql/ @eladkal
+/providers/src/airflow/providers/dbt/cloud/ @josh-fell
+/providers/src/airflow/providers/edge @jscheffl
+/providers/src/airflow/providers/hashicorp/ @hussein-awala
+/providers/src/airflow/providers/openlineage/ @mobuchowski
+/providers/src/airflow/providers/slack/ @eladkal
+/providers/src/airflow/providers/smtp/ @hussein-awala
+/providers/src/airflow/providers/snowflake/ @potiuk @mik-laj
+/providers/src/airflow/providers/apache/iceberg/ @Fokko
+/providers/tests/apache/iceberg/ @Fokko
 /docs/apache-airflow-providers-amazon/ @eladkal @o-nikolas
 /docs/apache-airflow-providers-cncf-kubernetes @jedcunningham
 /docs/apache-airflow-providers-common-sql/ @eladkal
 /docs/apache-airflow-providers-openlineage/ @mobuchowski
 /docs/apache-airflow-providers-slack/ @eladkal
-/tests/providers/amazon/ @eladkal @o-nikolas
-/tests/providers/common/sql/ @eladkal
-/tests/providers/openlineage/ @mobuchowski
-/tests/providers/slack/ @eladkal
+/providers/tests/amazon/ @eladkal @o-nikolas
+/providers/tests/common/sql/ @eladkal
+/providers/tests/openlineage/ @mobuchowski
+/providers/tests/slack/ @eladkal
 /tests/system/providers/amazon/ @eladkal @o-nikolas
 
 # Dev tools
@@ -108,6 +109,6 @@ ISSUE_TRIAGE_PROCESS.rst @eladkal
 
 # AIP-58 - Object Storage
 /airflow/io/ @bolkedebruin
-/airflow/providers/**/fs/ @bolkedebruin
-/airflow/providers/common/io/ @bolkedebruin
+/providers/src/airflow/providers/**/fs/ @bolkedebruin
+/providers/src/airflow/providers/common/io/ @bolkedebruin
 /docs/apache-airflow/core-concepts/objectstorage.rst @bolkedebruin

.github/workflows/additional-ci-image-checks.yml

@@ -84,6 +84,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to debug resources (true/false)"
         required: true
         type: string
+      use-uv:
+        description: "Whether to use uv to build the image (true/false)"
+        required: true
+        type: string
 jobs:
   # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the
   # tests to complete - it is run very early in the build process for "main" merges in order to refresh
@@ -113,7 +117,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
+      use-uv: ${{ inputs.use-uv}}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
@@ -170,7 +174,7 @@ jobs:
 #      platform: "linux/arm64"
 #      branch: ${{ inputs.branch }}
 #      constraints-branch: ${{ inputs.constraints-branch }}
-#      use-uv: "true"
+#      use-uv: ${{ inputs.use-uv}}
 #      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
 #      docker-cache: ${{ inputs.docker-cache }}
 #      disable-airflow-repo-cache:  ${{ inputs.disable-airflow-repo-cache }}

.github/workflows/build-images.yml

@@ -72,6 +72,7 @@ jobs:
       docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
       default-branch: ${{ steps.selective-checks.outputs.default-branch }}
       disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }}
+      force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
       runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }}
       runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }}
@@ -203,7 +204,7 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -248,7 +249,7 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: linux/amd64
       python-versions: ${{ needs.build-info.outputs.python-versions }}

.github/workflows/ci.yml

@@ -77,6 +77,7 @@ jobs:
       default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
       default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
       default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
+      force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
       parallel-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
@@ -205,7 +206,7 @@ jobs:
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
@@ -271,6 +272,7 @@ jobs:
       latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
 
 
   generate-constraints:
@@ -556,7 +558,7 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}

.github/workflows/finalize-tests.yml

@@ -149,7 +149,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}

.pre-commit-config.yaml

@@ -234,7 +234,7 @@ repos:
         additional_dependencies: ["libcst>=1.1.0"]
         files: ^(providers/src/)?airflow/.*/(sensors|operators)/.*\.py$
   - repo: https://github.com/asottile/blacken-docs
-    rev: 1.19.0
+    rev: 1.19.1
     hooks:
       - id: blacken-docs
         name: Run black on docs

Dockerfile

@@ -49,8 +49,13 @@ ARG AIRFLOW_VERSION="2.10.2"
 
 ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
 
-ARG AIRFLOW_PIP_VERSION=24.2
-ARG AIRFLOW_UV_VERSION=0.4.26
+
+# You can swap comments between those two args to test pip from the main version
+# When you attempt to test if the version of `pip` from specified branch works for our builds
+# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
+ARG AIRFLOW_PIP_VERSION=24.3.1
+# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
+ARG AIRFLOW_UV_VERSION=0.4.27
 ARG AIRFLOW_USE_UV="false"
 ARG UV_HTTP_TIMEOUT="300"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
@@ -615,7 +620,7 @@ function common::install_packaging_tools() {
         echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"
         echo
         pip install --root-user-action ignore --disable-pip-version-check --upgrade pip
-    elif [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
+    elif [[ ! ${AIRFLOW_PIP_VERSION} =~ ^[0-9].* ]]; then
         echo
         echo "${COLOR_BLUE}Installing pip version from spec ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
         echo
@@ -628,7 +633,6 @@ function common::install_packaging_tools() {
             echo
             echo "${COLOR_BLUE}(Re)Installing pip version: ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
             echo
-            # shellcheck disable=SC2086
             pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
         fi
     fi
@@ -637,7 +641,7 @@ function common::install_packaging_tools() {
         echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
         echo
         pip install --root-user-action ignore --disable-pip-version-check --upgrade uv
-    elif [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
+    elif [[ ! ${AIRFLOW_UV_VERSION} =~ ^[0-9].* ]]; then
         echo
         echo "${COLOR_BLUE}Installing uv version from spec ${AIRFLOW_UV_VERSION}${COLOR_RESET}"
         echo

Dockerfile.ci

@@ -561,7 +561,7 @@ function common::install_packaging_tools() {
         echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"
         echo
         pip install --root-user-action ignore --disable-pip-version-check --upgrade pip
-    elif [[ ! ${AIRFLOW_PIP_VERSION} =~ [0-9.]* ]]; then
+    elif [[ ! ${AIRFLOW_PIP_VERSION} =~ ^[0-9].* ]]; then
         echo
         echo "${COLOR_BLUE}Installing pip version from spec ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
         echo
@@ -574,7 +574,6 @@ function common::install_packaging_tools() {
             echo
             echo "${COLOR_BLUE}(Re)Installing pip version: ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
             echo
-            # shellcheck disable=SC2086
             pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
         fi
     fi
@@ -583,7 +582,7 @@ function common::install_packaging_tools() {
         echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
         echo
         pip install --root-user-action ignore --disable-pip-version-check --upgrade uv
-    elif [[ ! ${AIRFLOW_UV_VERSION} =~ [0-9.]* ]]; then
+    elif [[ ! ${AIRFLOW_UV_VERSION} =~ ^[0-9].* ]]; then
         echo
         echo "${COLOR_BLUE}Installing uv version from spec ${AIRFLOW_UV_VERSION}${COLOR_RESET}"
         echo
@@ -1362,8 +1361,12 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
 
 # Here we fix the versions so all subsequent commands will use the versions
 # from the sources
-ARG AIRFLOW_PIP_VERSION=24.2
-ARG AIRFLOW_UV_VERSION=0.4.26
+# You can swap comments between those two args to test pip from the main version
+# When you attempt to test if the version of `pip` from specified branch works for our builds
+# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
+ARG AIRFLOW_PIP_VERSION=24.3.1
+# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
+ARG AIRFLOW_UV_VERSION=0.4.27
 
 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
     AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION}

INTHEWILD.md

@@ -158,6 +158,7 @@ Currently, **officially** using Airflow:
 1. [Cryptalizer.com](https://www.cryptalizer.com/)
 1. [Currency](https://www.gocurrency.com/) [[@FCLI](https://github.com/FCLI) & [@alexbegg](https://github.com/alexbegg)]
 1. [Custom Ink](https://www.customink.com/) [[@david-dalisay](https://github.com/david-dalisay), [@dmartin11](https://github.com/dmartin11) & [@mpeteuil](https://github.com/mpeteuil)]
+1. [Cyberdino](https://www.cyberdino.io) [[@cyberdino-io](https://github.com/cyberdino-io)]
 1. [Cyscale](https://cyscale.com) [[@ocical](https://github.com/ocical)]
 1. [Dailymotion](http://www.dailymotion.com/fr) [[@germaintanguy](https://github.com/germaintanguy) & [@hc](https://github.com/hc)]
 1. [DANA](https://www.dana.id/) [[@imamdigmi](https://github.com/imamdigmi)]
@@ -383,7 +384,7 @@ Currently, **officially** using Airflow:
 1. [Paxful](https://paxful.com) [[@ne1r0n](https://github.com/ne1r0n)]
 1. [PayFit](https://payfit.com) [[@pcorbel](https://github.com/pcorbel)]
 1. [PAYMILL](https://www.paymill.com/) [[@paymill](https://github.com/paymill) & [@matthiashuschle](https://github.com/matthiashuschle)]
-1. [PayPal](https://www.paypal.com/) [[@r39132](https://github.com/r39132) & [@jhsenjaliya](https://github.com/jhsenjaliya)]
+1. [PayPal](https://www.paypal.com/) [[@kaddynator](https://github.com/kaddynator), [@r39132](https://github.com/r39132) & [@jhsenjaliya](https://github.com/jhsenjaliya)]
 1. [Pecan](https://www.pecan.ai) [[@ohadmata](https://github.com/ohadmata)]
 1. [Pernod-Ricard](https://www.pernod-ricard.com/) [[@romain-nio](https://github.com/romain-nio)]
 1. [PEXA](https://www.pexa.com.au/) [[@andriyfedorov](https://github.com/andriyfedorov)]

airflow/api_connexion/endpoints/dag_endpoint.py

@@ -134,7 +134,7 @@ def get_dags(
 
     try:
         dags_collection_schema = (
-            DAGCollectionSchema(only=[f"dags.{field}" for field in fields])
+            DAGCollectionSchema(only=[f"dags.{field}" for field in fields] + ["total_entries"])
             if fields
             else DAGCollectionSchema()
         )

airflow/api_connexion/endpoints/task_instance_endpoint.py

@@ -425,6 +425,7 @@ def get_task_instances_batch(session: Session = NEW_SESSION) -> APIResponse:
     except _UnsupportedOrderBy as e:
         raise BadRequest(detail=f"Ordering with {e.order_by!r} is not supported")
 
+    ti_query = ti_query.offset(data["page_offset"]).limit(data["page_limit"])
     task_instances = session.scalars(ti_query)
 
     return task_instance_collection_schema.dump(
@@ -533,9 +534,11 @@ def post_set_task_instances_state(*, dag_id: str, session: Session = NEW_SESSION
             detail=f"Task instance not found for task {task_id!r} on execution_date {execution_date}"
         )
 
-    if run_id and not session.get(
-        TI, {"task_id": task_id, "dag_id": dag_id, "run_id": run_id, "map_index": -1}
-    ):
+    select_stmt = select(TI).where(
+        TI.dag_id == dag_id, TI.task_id == task_id, TI.run_id == run_id, TI.map_index == -1
+    )
+
+    if run_id and not session.scalars(select_stmt).one_or_none():
         error_message = f"Task instance not found for task {task_id!r} on DAG run with ID {run_id!r}"
         raise NotFound(detail=error_message)
 
@@ -581,10 +584,12 @@ def patch_task_instance(
     if not dag.has_task(task_id):
         raise NotFound("Task not found", detail=f"Task {task_id!r} not found in DAG {dag_id!r}")
 
-    ti: TI | None = session.get(
-        TI, {"task_id": task_id, "dag_id": dag_id, "run_id": dag_run_id, "map_index": map_index}
+    select_stmt = select(TI).where(
+        TI.dag_id == dag_id, TI.task_id == task_id, TI.run_id == dag_run_id, TI.map_index == map_index
     )
 
+    ti: TI | None = session.scalars(select_stmt).one_or_none()
+
     if not ti:
         error_message = f"Task instance not found for task {task_id!r} on DAG run with ID {dag_run_id!r}"
         raise NotFound(detail=error_message)

airflow/api_connexion/openapi/v1.yaml

@@ -5038,6 +5038,15 @@ components:
     ListTaskInstanceForm:
       type: object
       properties:
+        page_offset:
+          type: integer
+          minimum: 0
+          description: The number of items to skip before starting to collect the result set.
+        page_limit:
+          type: integer
+          minimum: 1
+          default: 100
+          description: The numbers of items to return.
         dag_ids:
           type: array
           items: