Skip to content

Commit

Permalink
committing to park changes and wrap up other 1.7 items
Browse files Browse the repository at this point in the history
  • Loading branch information
mikealfare committed Oct 10, 2023
1 parent 6989ff3 commit 9a23f2f
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 16 deletions.
12 changes: 11 additions & 1 deletion dbt/adapters/bigquery/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import google.auth
import google.oauth2
import google.cloud.bigquery
from google.cloud.bigquery import AccessEntry, SchemaField
from google.cloud.bigquery import AccessEntry, SchemaField, Table
import google.cloud.exceptions

from dbt.adapters.bigquery import BigQueryColumn, BigQueryConnectionManager
Expand Down Expand Up @@ -555,6 +555,16 @@ def parse_partition_by(self, raw_partition_by: Any) -> Optional[PartitionConfig]
def get_table_ref_from_relation(self, relation: BaseRelation):
return self.connections.table_ref(relation.database, relation.schema, relation.identifier)

@available.parse(lambda *a, **k: True)
def get_table(self, relation: BigQueryRelation) -> Optional[Table]:
try:
table = self.connections.get_bq_table(
database=relation.database, schema=relation.schema, identifier=relation.identifier
)
except google.cloud.exceptions.NotFound:
table = None
return table

def _update_column_dict(self, bq_column_dict, dbt_columns, parent=""):
"""
Helper function to recursively traverse the schema of a table in the
Expand Down
5 changes: 1 addition & 4 deletions dbt/adapters/bigquery/relation_configs/materialized_view.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,7 @@ def parse_relation_results(cls, relation_results: RelationResults) -> Dict[str,

# optional
if partition_by := relation_results.get("partition_by"):
if len(partition_by) > 0:
config_dict.update(
{"partition": PartitionConfig.parse_relation_results(partition_by[0])}
)
config_dict.update({"partition": PartitionConfig.parse_relation_results(partition_by)}) # type: ignore

cluster_by: agate.Table = relation_results.get("cluster_by") # type: ignore
if len(cluster_by) > 0:
Expand Down
20 changes: 11 additions & 9 deletions dbt/adapters/bigquery/relation_configs/partition.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from dataclasses import dataclass
from typing import Any, Dict, List, Optional

import agate
from dbt.contracts.graph.nodes import ModelNode
from dbt.dataclass_schema import dbtClassMixin, ValidationError
import dbt.exceptions
from dbt.adapters.relation_configs import RelationConfigChange
from google.cloud.bigquery.table import Table


@dataclass
Expand Down Expand Up @@ -108,21 +108,23 @@ def parse_model_node(cls, model_node: ModelNode) -> Dict[str, Any]:
return model_node.config.extra.get("partition_by")

@classmethod
def parse_relation_results(cls, describe_relation_results: agate.Row) -> Dict[str, Any]:
def parse_relation_results(cls, bq_table: Table) -> Dict[str, Any]:
"""
Parse the results of a describe query into a raw config for `PartitionConfig.parse`
Parse the results of a BQ Table object into a raw config for `PartitionConfig.parse`
"""
range_partitioning = bq_table.range_partitioning
time_partitioning = bq_table.time_partitioning
config_dict = {
"field": describe_relation_results.get("partition_column_name"),
"data_type": describe_relation_results.get("partition_data_type"),
"granularity": describe_relation_results.get("partition_type"),
"field": time_partitioning.field,
"data_type": "",
"granularity": time_partitioning.type_,
}

# combine range fields into dictionary, like the model config
range_dict = {
"start": describe_relation_results.get("partition_start"),
"end": describe_relation_results.get("partition_end"),
"interval": describe_relation_results.get("partition_interval"),
"start": range_partitioning.range_.start,
"end": range_partitioning.range_.end,
"interval": range_partitioning.range_.interval,
}
config_dict.update({"range": range_dict})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,6 @@


{% macro bigquery__describe_partition(relation) %}
{% set _sql = bigquery__get_describe_partition_sql(relation) %}
{% do return(run_query(_sql)) %}
{% set bq_relation = adapter.connections.get_bq_table(relation.database, relation.schema, relation.identifier) %}
{% do return(bq_relation) %}
{% endmacro %}
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
{% macro bigquery__describe_materialized_view(relation) %}
{% set bq_relation = adapter.get_table(relation) %}
{% do return(bq_relation) %}
{% endmacro %}


{% macro bigquery__describe_materialized_view_sql(relation) %}
{%- set _materialized_view_sql -%}
select
table_name,
Expand All @@ -16,6 +22,7 @@

{% do return({
'materialized_view': _materialized_view,
'partition_by': bigquery__describe_partition(relation),
'cluster_by': _cluster_by,
'options': _options
}) %}
Expand Down

0 comments on commit 9a23f2f

Please sign in to comment.