OpenLineage: accept whole config when instantiating OpenLineageClient. (

#43740)

Signed-off-by: Jakub Dardzinski <kuba0221@gmail.com>

generated/provider_dependencies.json

@@ -976,8 +976,8 @@
       "apache-airflow-providers-common-sql>=1.6.0",
       "apache-airflow>=2.8.0",
       "attrs>=22.2",
-      "openlineage-integration-common>=1.22.0,<1.24.0",
-      "openlineage-python>=1.22.0,<1.24.0"
+      "openlineage-integration-common>=1.24.2",
+      "openlineage-python>=1.24.2"
     ],
     "devel-deps": [],
     "plugins": [

providers/src/airflow/providers/openlineage/plugins/adapter.py

@@ -84,7 +84,7 @@ def get_or_create_openlineage_client(self) -> OpenLineageClient:
                     "OpenLineage configuration found. Transport type: `%s`",
                     config.get("type", "no type provided"),
                 )
-                self._client = OpenLineageClient.from_dict(config=config)
+                self._client = OpenLineageClient(config=config)
             else:
                 self.log.debug(
                     "OpenLineage configuration not found directly in Airflow. "
@@ -98,9 +98,7 @@ def get_openlineage_config(self) -> dict | None:
         openlineage_config_path = conf.config_path(check_legacy_env_var=False)
         if openlineage_config_path:
             config = self._read_yaml_config(openlineage_config_path)
-            if config:
-                return config.get("transport", None)
-            self.log.debug("OpenLineage config file is empty: `%s`", openlineage_config_path)
+            return config
         else:
             self.log.debug("OpenLineage config_path configuration not found.")
 
@@ -109,7 +107,7 @@ def get_openlineage_config(self) -> dict | None:
         if not transport_config:
             self.log.debug("OpenLineage transport configuration not found.")
             return None
-        return transport_config
+        return {"transport": transport_config}
 
     @staticmethod
     def _read_yaml_config(path: str) -> dict | None:

providers/src/airflow/providers/openlineage/provider.yaml

@@ -54,10 +54,8 @@ dependencies:
   - apache-airflow-providers-common-sql>=1.6.0
   - apache-airflow-providers-common-compat>=1.2.1
   - attrs>=22.2
-  # Temporary limiting Open Lineage version to 1.23.0 due to compatibility issues with 1.24.0
-  # Causing test failures, waiting for 1.24.1 to be released to fix the issue
-  - openlineage-integration-common>=1.22.0,<1.24.0
-  - openlineage-python>=1.22.0,<1.24.0
+  - openlineage-integration-common>=1.24.2
+  - openlineage-python>=1.24.2
 
 integrations:
   - integration-name: OpenLineage

providers/tests/openlineage/plugins/test_adapter.py

@@ -81,17 +81,6 @@ def test_create_client_from_config_with_options():
     assert client.transport.url == "http://ol-api:5000"
 
 
-@conf_vars(
-    {
-        ("openlineage", "transport"): '{"url": "http://ol-api:5000",'
-        ' "auth": {"type": "api_key", "apiKey": "api-key"}}'
-    }
-)
-def test_fails_to_create_client_without_type():
-    with pytest.raises(KeyError):
-        OpenLineageAdapter().get_or_create_openlineage_client()
-
-
 def test_create_client_from_yaml_config():
     current_folder = pathlib.Path(__file__).parent.resolve()
     yaml_config = str((current_folder / "openlineage_configs" / "http.yaml").resolve())