Fix completion/linting/type checking with VSCode/pyright (apache#43899)

Pyright (the type engine powering VSCode's python extension) doesn't treat
`airflow` as a namespace package because of the `airflow/__init__.py` and it
doesn't want to/can't support detecting the `__path__ = ...` method of making
it an explicit namespace package, so we are left with no option but to create
Yet Another Stub File.

Tested by pytting `reveal_type(FAB_VERSION); reveal_type(TaskSDKDag)` inside
`_upgrade_outdated_dag_access_control` in `airflow/model/dag.py` -- before
this change it was reporting both as Unknown.

And to continue to keep Pycharm happy we have to have the `__path__` stanzas
without future annotations in all the "empty" files.

Ugly, but at least it works

Dockerfile

@@ -890,7 +890,7 @@ function install_airflow() {
 
         # Similarly we need _a_ file for task_sdk too
         mkdir -p ./task_sdk/src/airflow/sdk/
-        touch ./task_sdk/src/airflow/__init__.py
+        touch ./task_sdk/src/airflow/sdk/__init__.py
 
         trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT
 

Dockerfile.ci

@@ -660,7 +660,7 @@ function install_airflow() {
 
         # Similarly we need _a_ file for task_sdk too
         mkdir -p ./task_sdk/src/airflow/sdk/
-        touch ./task_sdk/src/airflow/__init__.py
+        touch ./task_sdk/src/airflow/sdk/__init__.py
 
         trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT
 

providers/src/airflow/__init__.py

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pycharm needs to see this line. VSCode/pyright doesn't care about it, but this file needs to exist
+# https://github.com/microsoft/pyright/issues/9439#issuecomment-2468990559
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore

providers/src/airflow/providers/__init__.py

@@ -16,13 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# We do not use "from __future__ import annotations" here because it is not supported
-# by Pycharm when we want to make sure all imports in airflow work from namespace packages
-# Adding it automatically is excluded in pyproject.toml via I002 ruff rule exclusion
-
-# Make `airflow` a namespace package, supporting installing
-# airflow.providers.* in different locations (i.e. one in site, and one in user
-# lib.)  This is required by some IDEs to resolve the import paths.
-#
-# Note: this file is not installed or distributed in any distribution!
+# Pycharm needs to see this line. VSCode/pyright doesn't care about it, but this file needs to exist
+# https://github.com/microsoft/pyright/issues/9439#issuecomment-2468990559
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore

pyproject.toml

@@ -317,6 +317,7 @@ testing = ["dev", "providers.tests", "task_sdk.tests", "tests_common", "tests"]
 "airflow/models/__init__.py" = ["F401", "TCH004"]
 "airflow/models/sqla_models.py" = ["F401"]
 "providers/src/airflow/providers/__init__.py" = ["I002"]
+"providers/src/airflow/__init__.py" = ["I002"]
 
 # The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
 # needed for the test to work

scripts/docker/install_airflow.sh

@@ -54,7 +54,7 @@ function install_airflow() {
 
         # Similarly we need _a_ file for task_sdk too
         mkdir -p ./task_sdk/src/airflow/sdk/
-        touch ./task_sdk/src/airflow/__init__.py
+        touch ./task_sdk/src/airflow/sdk/__init__.py
 
         trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT
 

task_sdk/pyproject.toml

@@ -33,6 +33,8 @@ build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/airflow"]
+# This file only exists to make pyright/VSCode happy, don't ship it
+exclude = ["src/airflow/__init__.py"]
 
 [tool.ruff]
 extend = "../pyproject.toml"
@@ -46,6 +48,9 @@ namespace-packages = ["src/airflow"]
 
 "src/airflow/sdk/__init__.py" = ["TCH004"]
 
+# Pycharm barfs if this "stub" file has future imports
+"src/airflow/__init__.py" = ["I002"]
+
 [tool.uv]
 dev-dependencies = [
     "kgb>=7.1.1",

task_sdk/src/airflow/__init__.py

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pycharm needs to see this line. VSCode/pyright doesn't care about it, but this file needs to exist
+# https://github.com/microsoft/pyright/issues/9439#issuecomment-2468990559
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore