Skip to content

Commit

Permalink
fix(ingestion/airflow-plugin): fix AthenaOperator extraction
Browse files Browse the repository at this point in the history
The GenericSqlExtractor which is currently by the DataHub Airflow plugin
to extract lineage information does not properly support the
AthenaOperator and crashes with "AttributeError: 'AthenaOperator' object
has no attribute 'sql'". This patch introduces a AthenaOperatorExtractor
following the BigQueryInsertJobOperatorExtractor example to fix support
for the AthenaOperator.
  • Loading branch information
steffengr committed Nov 14, 2024
1 parent ab0b0a2 commit b9e369e
Showing 1 changed file with 23 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ def __init__(self):
"BigQueryOperator",
"BigQueryExecuteQueryOperator",
# Athena also does something similar.
"AthenaOperator",
"AWSAthenaOperator",
# Additional types that OL doesn't support. This is only necessary because
# on older versions of Airflow, these operators don't inherit from SQLExecuteQueryOperator.
Expand All @@ -59,6 +58,8 @@ def __init__(self):
for operator in _sql_operator_overrides:
self.task_to_extractor.extractors[operator] = GenericSqlExtractor

self.task_to_extractor.extractors["AthenaOperator"] = AthenaOperatorExtractor

self.task_to_extractor.extractors[
"BigQueryInsertJobOperator"
] = BigQueryInsertJobOperatorExtractor
Expand Down Expand Up @@ -276,6 +277,27 @@ def extract(self) -> Optional[TaskMetadata]:
)


class AthenaOperatorExtractor(BaseExtractor):
def extract(self) -> Optional[TaskMetadata]:
from airflow.providers.amazon.aws.operators.athena import (
AthenaOperator, # type: ignore
)

operator: "AthenaOperator" = self.operator
sql = operator.query
if not sql:
self.log.warning("No query found in AthenaOperator")
return None

return _parse_sql_into_task_metadata(
self,
sql,
platform="athena",
default_database=None,
default_schema=None,
)


def _snowflake_default_schema(self: "SnowflakeExtractor") -> Optional[str]:
if hasattr(self.operator, "schema") and self.operator.schema is not None:
return self.operator.schema
Expand Down

0 comments on commit b9e369e

Please sign in to comment.