From be7ce069bb5c7ffcce07f25c68aeeef9966a54fd Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Wed, 8 Nov 2023 10:40:47 -0500 Subject: [PATCH 01/22] Add allow_non_incremental_definition option to BigQuery materialized views --- dbt/adapters/bigquery/impl.py | 1 + dbt/adapters/bigquery/relation_configs/_options.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 7d9b003b8..6ee9eb55f 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -92,6 +92,7 @@ class BigqueryConfig(AdapterConfig): enable_refresh: Optional[bool] = None refresh_interval_minutes: Optional[int] = None max_staleness: Optional[str] = None + allow_non_incremental_definition: Optional[bool] = None class BigQueryAdapter(BaseAdapter): diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index 72f9d73e6..4c373b06c 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -21,6 +21,7 @@ class BigQueryOptionsConfig(BigQueryBaseRelationConfig): refresh_interval_minutes: Optional[float] = 30 expiration_timestamp: Optional[datetime] = None max_staleness: Optional[str] = None + allow_non_incremental_definition: Optional[bool] = None kms_key_name: Optional[str] = None description: Optional[str] = None labels: Optional[Dict[str, str]] = None @@ -57,6 +58,7 @@ def array(x): "refresh_interval_minutes": numeric, "expiration_timestamp": interval, "max_staleness": interval, + "allow_non_incremental_definition": boolean, "kms_key_name": string, "description": escaped_string, "labels": array, @@ -84,6 +86,7 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "BigQueryOptionsConfig": "refresh_interval_minutes": float_setting, "expiration_timestamp": None, "max_staleness": None, + "allow_non_incremental_definition": bool_setting, "kms_key_name": None, "description": None, "labels": None, @@ -114,6 +117,7 @@ def parse_model_node(cls, model_node: ModelNode) -> Dict[str, Any]: "refresh_interval_minutes", "expiration_timestamp", "max_staleness", + "allow_non_incremental_definition", "kms_key_name", "description", "labels", @@ -137,6 +141,7 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: "refresh_interval_minutes": table.mview_refresh_interval.seconds / 60, "expiration_timestamp": table.expires, "max_staleness": None, + "allow_non_incremental_definition": table._properties.get("materializedView", {}).get("allowNonIncrementalDefinition"), "description": table.description, } From 85e3426019dbf804b049d5328e2ab37ab07b450c Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Wed, 8 Nov 2023 19:05:15 -0500 Subject: [PATCH 02/22] Add changelog --- .changes/unreleased/Features-20231108-140752.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20231108-140752.yaml diff --git a/.changes/unreleased/Features-20231108-140752.yaml b/.changes/unreleased/Features-20231108-140752.yaml new file mode 100644 index 000000000..2a9a41a07 --- /dev/null +++ b/.changes/unreleased/Features-20231108-140752.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support allow_non_incremental_definition option in BigQuery materialized views +time: 2023-11-08T14:07:52.28972-05:00 +custom: + Author: bnaul + Issue: "672" From c976c81c229f58892651cc1ae9c7f5135fab2379 Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Wed, 8 Nov 2023 19:18:02 -0500 Subject: [PATCH 03/22] Fix default value for flag --- dbt/adapters/bigquery/relation_configs/_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index 4c373b06c..e7a434da7 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -21,7 +21,7 @@ class BigQueryOptionsConfig(BigQueryBaseRelationConfig): refresh_interval_minutes: Optional[float] = 30 expiration_timestamp: Optional[datetime] = None max_staleness: Optional[str] = None - allow_non_incremental_definition: Optional[bool] = None + allow_non_incremental_definition: Optional[bool] = False kms_key_name: Optional[str] = None description: Optional[str] = None labels: Optional[Dict[str, str]] = None From b3e9eea12d0650ed72186e2e1bdaf13a9ae9774e Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Wed, 8 Nov 2023 19:18:07 -0500 Subject: [PATCH 04/22] Set flag in test --- tests/functional/adapter/materialized_view_tests/_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/materialized_view_tests/_files.py b/tests/functional/adapter/materialized_view_tests/_files.py index 86714036a..9f8fe14d3 100644 --- a/tests/functional/adapter/materialized_view_tests/_files.py +++ b/tests/functional/adapter/materialized_view_tests/_files.py @@ -36,7 +36,8 @@ cluster_by=["id", "value"], enable_refresh=True, refresh_interval_minutes=60, - max_staleness="INTERVAL 45 MINUTE" + max_staleness="INTERVAL 45 MINUTE", + allow_non_incremental_definition=True ) }} select id, From 97332e6c5451d5c35ca1dbab5c1fd5ad02ac6bc9 Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Thu, 9 Nov 2023 20:33:32 -0500 Subject: [PATCH 05/22] Recreate if allow_non_incremental_definition changed --- dbt/adapters/bigquery/relation.py | 10 +++++++++- dbt/adapters/bigquery/relation_configs/_options.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index c14dba238..c4977be8d 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -78,8 +78,16 @@ def materialized_view_config_changeset( new_materialized_view = cls.materialized_view_from_model_node(runtime_config.model) if new_materialized_view.options != existing_materialized_view.options: + # allow_non_incremental_definition cannot be changed via ALTER, must recreate + if ( + new_materialized_view.options.allow_non_incremental_definition + == existing_materialized_view.options.allow_non_incremental_definition + ): + action = RelationConfigChangeAction.alter + else: + action = RelationConfigChangeAction.create config_change_collection.options = BigQueryOptionsConfigChange( - action=RelationConfigChangeAction.alter, + action=action, context=new_materialized_view.options, ) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index e7a434da7..a36babcf2 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -8,6 +8,7 @@ from dbt.adapters.bigquery.relation_configs._base import BigQueryBaseRelationConfig from dbt.adapters.bigquery.utility import bool_setting, float_setting, sql_escape +from dbt.adapters.relation_configs import RelationConfigChangeAction @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -160,4 +161,4 @@ class BigQueryOptionsConfigChange(RelationConfigChange): @property def requires_full_refresh(self) -> bool: - return False + return self.action != RelationConfigChangeAction.alter From e75ccacab9a740448af7f917e2fb4b7da6282f19 Mon Sep 17 00:00:00 2001 From: Brett Naul Date: Sun, 12 Nov 2023 08:09:33 -0500 Subject: [PATCH 06/22] Parse maxStaleness --- dbt/adapters/bigquery/relation_configs/_options.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index a36babcf2..924e1b98d 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -141,7 +141,11 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: "enable_refresh": table.mview_enable_refresh, "refresh_interval_minutes": table.mview_refresh_interval.seconds / 60, "expiration_timestamp": table.expires, - "max_staleness": None, + "max_staleness": ( + f"INTERVAL '{table._properties.get('maxStaleness')}' YEAR TO SECOND" + if table._properties.get('maxStaleness') + else None + ), "allow_non_incremental_definition": table._properties.get("materializedView", {}).get("allowNonIncrementalDefinition"), "description": table.description, } From 40d3e55d617f26493c554488a03697df2eb27fa1 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 11 Jun 2024 17:03:31 -0400 Subject: [PATCH 07/22] implement feedback from #1011 --- dbt/adapters/bigquery/connections.py | 2 +- dbt/adapters/bigquery/relation.py | 8 ++++---- dbt/adapters/bigquery/relation_configs/_options.py | 10 ++++++---- .../test_materialized_view_changes.py | 2 ++ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index 4a3feae48..856b39bb6 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -320,7 +320,7 @@ def format_rows_number(self, rows_number): return f"{rows_number:3.1f}{unit}".strip() @classmethod - def get_google_credentials(cls, profile_credentials) -> GoogleCredentials: + def get_google_credentials(cls, profile_credentials) -> GoogleCredentials.Credentials: method = profile_credentials.method creds = GoogleServiceAccountCredentials.Credentials diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 99aa2036c..34556ead1 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -90,14 +90,14 @@ def materialized_view_config_changeset( new_materialized_view = cls.materialized_view_from_relation_config(relation_config) if new_materialized_view.options != existing_materialized_view.options: - # allow_non_incremental_definition cannot be changed via ALTER, must recreate + # allow_non_incremental_definition cannot be changed via an ALTER statement if ( new_materialized_view.options.allow_non_incremental_definition - == existing_materialized_view.options.allow_non_incremental_definition + != existing_materialized_view.options.allow_non_incremental_definition ): - action = RelationConfigChangeAction.alter - else: action = RelationConfigChangeAction.create + else: + action = RelationConfigChangeAction.alter config_change_collection.options = BigQueryOptionsConfigChange( action=action, context=new_materialized_view.options, diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index 2f51acedf..6c1bdca96 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -23,7 +23,7 @@ class BigQueryOptionsConfig(BigQueryBaseRelationConfig): refresh_interval_minutes: Optional[float] = 30 expiration_timestamp: Optional[datetime] = None max_staleness: Optional[str] = None - allow_non_incremental_definition: Optional[bool] = False + allow_non_incremental_definition: Optional[bool] = None kms_key_name: Optional[str] = None description: Optional[str] = None labels: Optional[Dict[str, str]] = None @@ -146,10 +146,12 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: "expiration_timestamp": table.expires, "max_staleness": ( f"INTERVAL '{table._properties.get('maxStaleness')}' YEAR TO SECOND" - if table._properties.get('maxStaleness') + if table._properties.get("maxStaleness") else None ), - "allow_non_incremental_definition": table._properties.get("materializedView", {}).get("allowNonIncrementalDefinition"), + "allow_non_incremental_definition": table._properties.get("materializedView", {}).get( + "allowNonIncrementalDefinition" + ), "description": table.description, } @@ -168,4 +170,4 @@ class BigQueryOptionsConfigChange(RelationConfigChange): @property def requires_full_refresh(self) -> bool: - return self.action != RelationConfigChangeAction.alter + return self.action == RelationConfigChangeAction.create diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py index c821c68fc..7146e92c0 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py @@ -19,6 +19,8 @@ def check_start_state(project, materialized_view): assert isinstance(results, BigQueryMaterializedViewConfig) assert results.options.enable_refresh is True assert results.options.refresh_interval_minutes == 60 + assert results.options.max_staleness == "INTERVAL 45 MINUTE" + assert results.options.allow_non_incremental_definition is True assert results.partition.field == "record_valid_date" assert results.partition.data_type == "datetime" assert results.partition.granularity == "day" From ce9724bb89bd28a4606f1d0833cb614acc9706c5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jun 2024 11:51:02 -0400 Subject: [PATCH 08/22] move options config change logic next to options config, reduce to only changed options --- dbt/adapters/bigquery/relation.py | 15 ++++----------- .../bigquery/relation_configs/_options.py | 19 +++++++++++++++++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 34556ead1..38065dff6 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -90,17 +90,10 @@ def materialized_view_config_changeset( new_materialized_view = cls.materialized_view_from_relation_config(relation_config) if new_materialized_view.options != existing_materialized_view.options: - # allow_non_incremental_definition cannot be changed via an ALTER statement - if ( - new_materialized_view.options.allow_non_incremental_definition - != existing_materialized_view.options.allow_non_incremental_definition - ): - action = RelationConfigChangeAction.create - else: - action = RelationConfigChangeAction.alter - config_change_collection.options = BigQueryOptionsConfigChange( - action=action, - context=new_materialized_view.options, + # get an options change object with only the options that have changed + config_change_collection.options = BigQueryOptionsConfigChange.from_options_configs( + new_materialized_view.options, + existing_materialized_view.options, ) if new_materialized_view.partition != existing_materialized_view.partition: diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index 6c1bdca96..604fe78e2 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import asdict, dataclass from datetime import datetime, timedelta from typing import Any, Dict, Optional @@ -170,4 +170,19 @@ class BigQueryOptionsConfigChange(RelationConfigChange): @property def requires_full_refresh(self) -> bool: - return self.action == RelationConfigChangeAction.create + # allow_non_incremental_definition cannot be changed via an ALTER statement + return self.context.allow_non_incremental_definition is not None + + @classmethod + def from_options_configs( + cls, new_options: BigQueryOptionsConfig, existing_options: BigQueryOptionsConfig + ) -> Self: + new_options_dict = asdict(new_options) + existing_options_dict = asdict(existing_options) + option_diffs_dict = { + k: v + for k, v in new_options_dict.items() + if new_options_dict[k] != existing_options_dict[k] + } + option_diffs = BigQueryOptionsConfig.from_dict(option_diffs_dict) + return cls(action=RelationConfigChangeAction.alter, context=option_diffs) From d62a80daa685c56ce2d196f14a437232aaf5d786 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jun 2024 11:52:33 -0400 Subject: [PATCH 09/22] fix the expected value for max_staleness --- .../materialized_view_tests/test_materialized_view_changes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py index 7146e92c0..2d47f260f 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py @@ -19,7 +19,7 @@ def check_start_state(project, materialized_view): assert isinstance(results, BigQueryMaterializedViewConfig) assert results.options.enable_refresh is True assert results.options.refresh_interval_minutes == 60 - assert results.options.max_staleness == "INTERVAL 45 MINUTE" + assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" assert results.options.allow_non_incremental_definition is True assert results.partition.field == "record_valid_date" assert results.partition.data_type == "datetime" From ee54a7c6faef5768c1a8e13b925a587bfb752ee2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jun 2024 11:55:52 -0400 Subject: [PATCH 10/22] move setup into basic tests since it already exists in the changes base --- .../adapter/materialized_view_tests/_mixin.py | 21 +------------- .../test_materialized_view.py | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/tests/functional/adapter/materialized_view_tests/_mixin.py b/tests/functional/adapter/materialized_view_tests/_mixin.py index 4eb98e047..f2838a586 100644 --- a/tests/functional/adapter/materialized_view_tests/_mixin.py +++ b/tests/functional/adapter/materialized_view_tests/_mixin.py @@ -5,12 +5,7 @@ from dbt.adapters.base.relation import BaseRelation from dbt.adapters.contracts.relation import RelationType from dbt.tests.adapter.materialized_view.files import MY_TABLE, MY_VIEW -from dbt.tests.util import ( - get_connection, - get_model_file, - run_dbt, - set_model_file, -) +from dbt.tests.util import get_connection from tests.functional.adapter.materialized_view_tests import _files @@ -45,20 +40,6 @@ def my_other_base_table(self, project) -> BaseRelation: type=RelationType.Table, ) - @pytest.fixture(scope="function", autouse=True) - def setup(self, project, my_base_table, my_other_base_table, my_materialized_view): # type: ignore - run_dbt(["seed"]) - run_dbt(["run", "--full-refresh"]) - - # the tests touch these files, store their contents in memory - initial_model = get_model_file(project, my_materialized_view) - - yield - - # and then reset them after the test runs - set_model_file(project, my_materialized_view, initial_model) - project.run_sql(f"drop schema if exists {project.test_schema} cascade") - @pytest.fixture(scope="class", autouse=True) def seeds(self): return {"my_seed.csv": _files.MY_SEED} diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view.py index 4e980c2e4..3cb535eff 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view.py @@ -2,12 +2,27 @@ from dbt.tests.util import run_dbt from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic +from dbt.tests.util import get_model_file, set_model_file from tests.functional.adapter.materialized_view_tests._mixin import BigQueryMaterializedViewMixin from tests.functional.adapter.materialized_view_tests import _files class TestBigqueryMaterializedViewsBasic(BigQueryMaterializedViewMixin, MaterializedViewBasic): + @pytest.fixture(scope="function", autouse=True) + def setup(self, project, my_materialized_view): # type: ignore + run_dbt(["seed"]) + run_dbt(["run", "--full-refresh"]) + + # the tests touch these files, store their contents in memory + initial_model = get_model_file(project, my_materialized_view) + + yield + + # and then reset them after the test runs + set_model_file(project, my_materialized_view, initial_model) + project.run_sql(f"drop schema if exists {project.test_schema} cascade") + def test_view_replaces_materialized_view(self, project, my_materialized_view): """ We don't support replacing a view with another object in dbt-bigquery unless we use --full-refresh @@ -39,6 +54,20 @@ class TestMaterializedViewRerun: and cause unexpected scenarios. """ + @pytest.fixture(scope="function", autouse=True) + def setup(self, project, my_materialized_view): # type: ignore + run_dbt(["seed"]) + run_dbt(["run", "--full-refresh"]) + + # the tests touch these files, store their contents in memory + initial_model = get_model_file(project, my_materialized_view) + + yield + + # and then reset them after the test runs + set_model_file(project, my_materialized_view, initial_model) + project.run_sql(f"drop schema if exists {project.test_schema} cascade") + @pytest.fixture(scope="class", autouse=True) def models(self): return {"my_minimal_materialized_view.sql": _files.MY_MINIMAL_MATERIALIZED_VIEW} From 2caf436bd954c2c4793b4e013ed4fa8e2ad511ef Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jun 2024 11:57:28 -0400 Subject: [PATCH 11/22] remove setup from rerun class because it's not needed --- .../test_materialized_view.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view.py index 3cb535eff..ca2715769 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view.py @@ -54,20 +54,6 @@ class TestMaterializedViewRerun: and cause unexpected scenarios. """ - @pytest.fixture(scope="function", autouse=True) - def setup(self, project, my_materialized_view): # type: ignore - run_dbt(["seed"]) - run_dbt(["run", "--full-refresh"]) - - # the tests touch these files, store their contents in memory - initial_model = get_model_file(project, my_materialized_view) - - yield - - # and then reset them after the test runs - set_model_file(project, my_materialized_view, initial_model) - project.run_sql(f"drop schema if exists {project.test_schema} cascade") - @pytest.fixture(scope="class", autouse=True) def models(self): return {"my_minimal_materialized_view.sql": _files.MY_MINIMAL_MATERIALIZED_VIEW} From 4858eef77c37988177669144c24873a39ae4b306 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jun 2024 12:04:21 -0400 Subject: [PATCH 12/22] move the setup back into the mixin so that all relations are refreshed --- .../adapter/materialized_view_tests/_mixin.py | 28 +++++++++++++++++-- .../test_materialized_view.py | 15 ---------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/tests/functional/adapter/materialized_view_tests/_mixin.py b/tests/functional/adapter/materialized_view_tests/_mixin.py index f2838a586..474e1dbb0 100644 --- a/tests/functional/adapter/materialized_view_tests/_mixin.py +++ b/tests/functional/adapter/materialized_view_tests/_mixin.py @@ -5,8 +5,12 @@ from dbt.adapters.base.relation import BaseRelation from dbt.adapters.contracts.relation import RelationType from dbt.tests.adapter.materialized_view.files import MY_TABLE, MY_VIEW -from dbt.tests.util import get_connection - +from dbt.tests.util import ( + get_connection, + get_model_file, + run_dbt, + set_model_file, +) from tests.functional.adapter.materialized_view_tests import _files @@ -54,6 +58,26 @@ def models(self): "my_materialized_view.sql": _files.MY_MATERIALIZED_VIEW, } + @pytest.fixture(scope="function", autouse=True) + def setup(self, project, my_materialized_view): + """ + We overwrite this so that we get a `--full-refresh` of everything between tests, instead of just the MV + """ + # make sure the model in the data reflects the files each time + run_dbt(["seed"]) + run_dbt(["run", "--full-refresh"]) + + # the tests touch these files, store their contents in memory + initial_model = get_model_file(project, my_materialized_view) + + yield + + # and then reset them after the test runs + set_model_file(project, my_materialized_view, initial_model) + + # ensure clean slate each method + project.run_sql(f"drop schema if exists {project.test_schema} cascade") + @staticmethod def insert_record(project, table: BaseRelation, record: Tuple[int, int]) -> None: my_id, value = record diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view.py index ca2715769..4e980c2e4 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view.py @@ -2,27 +2,12 @@ from dbt.tests.util import run_dbt from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic -from dbt.tests.util import get_model_file, set_model_file from tests.functional.adapter.materialized_view_tests._mixin import BigQueryMaterializedViewMixin from tests.functional.adapter.materialized_view_tests import _files class TestBigqueryMaterializedViewsBasic(BigQueryMaterializedViewMixin, MaterializedViewBasic): - @pytest.fixture(scope="function", autouse=True) - def setup(self, project, my_materialized_view): # type: ignore - run_dbt(["seed"]) - run_dbt(["run", "--full-refresh"]) - - # the tests touch these files, store their contents in memory - initial_model = get_model_file(project, my_materialized_view) - - yield - - # and then reset them after the test runs - set_model_file(project, my_materialized_view, initial_model) - project.run_sql(f"drop schema if exists {project.test_schema} cascade") - def test_view_replaces_materialized_view(self, project, my_materialized_view): """ We don't support replacing a view with another object in dbt-bigquery unless we use --full-refresh From 856dbc24fef9e02a19fca4aff29f9846d296620e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 13:19:55 -0400 Subject: [PATCH 13/22] account for interaction between enable_refresh and allow_non_incremental_definition --- dbt/adapters/bigquery/relation_configs/_options.py | 8 +++++++- .../test_materialized_view_changes.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index 604fe78e2..144c49e9f 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -105,7 +105,13 @@ def formatted_setting(name: str) -> Any: # avoid picking up defaults on dependent options # e.g. don't set `refresh_interval_minutes` = 30 when the user has `enable_refresh` = False if kwargs_dict["enable_refresh"] is False: - kwargs_dict.update({"refresh_interval_minutes": None, "max_staleness": None}) + kwargs_dict.update( + { + "refresh_interval_minutes": None, + "max_staleness": None, + "allow_non_incremental_definition": None, + } + ) options: Self = super().from_dict(kwargs_dict) # type: ignore return options diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py index 2d47f260f..1a94b93ec 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py @@ -29,7 +29,9 @@ def check_start_state(project, materialized_view): @staticmethod def change_config_via_alter(project, materialized_view): initial_model = get_model_file(project, materialized_view) - new_model = initial_model.replace("enable_refresh=True", "enable_refresh=False") + new_model = initial_model.replace( + "refresh_interval_minutes=60", "refresh_interval_minutes=45" + ) set_model_file(project, materialized_view, new_model) @staticmethod @@ -37,9 +39,7 @@ def check_state_alter_change_is_applied(project, materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(materialized_view) assert isinstance(results, BigQueryMaterializedViewConfig) - # these change when run manually - assert results.options.enable_refresh is False - assert results.options.refresh_interval_minutes == 30 # BQ returns it to the default + assert results.options.refresh_interval_minutes == 45 @staticmethod def change_config_via_replace(project, materialized_view): @@ -65,6 +65,9 @@ def change_config_via_replace(project, materialized_view): initial_model.replace(old_partition, new_partition) .replace("'my_base_table'", "'my_other_base_table'") .replace('cluster_by=["id", "value"]', 'cluster_by="id"') + .replace( + "allow_non_incremental_definition=True", "allow_non_incremental_definition=False" + ) ) set_model_file(project, materialized_view, new_model) @@ -77,6 +80,7 @@ def check_state_replace_change_is_applied(project, materialized_view): assert results.partition.data_type == "int64" assert results.partition.range == {"start": 0, "end": 500, "interval": 50} assert results.cluster.fields == frozenset({"id"}) + assert results.options.allow_non_incremental_definition is False class TestBigQueryMaterializedViewChangesApply( From 64c397f64ed5554645960a67e4b43d794be5a6b5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 17:34:35 -0400 Subject: [PATCH 14/22] remove inadvertent format changes --- .../adapter/materialized_view_tests/_files.py | 2 +- .../adapter/materialized_view_tests/_mixin.py | 35 ++++++++----------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/tests/functional/adapter/materialized_view_tests/_files.py b/tests/functional/adapter/materialized_view_tests/_files.py index 9f8fe14d3..aa6803219 100644 --- a/tests/functional/adapter/materialized_view_tests/_files.py +++ b/tests/functional/adapter/materialized_view_tests/_files.py @@ -36,7 +36,7 @@ cluster_by=["id", "value"], enable_refresh=True, refresh_interval_minutes=60, - max_staleness="INTERVAL 45 MINUTE", + max_staleness="INTERVAL '0-0 0 0:45:0' YEAR TO SECOND", allow_non_incremental_definition=True ) }} select diff --git a/tests/functional/adapter/materialized_view_tests/_mixin.py b/tests/functional/adapter/materialized_view_tests/_mixin.py index 474e1dbb0..7a1ba4e94 100644 --- a/tests/functional/adapter/materialized_view_tests/_mixin.py +++ b/tests/functional/adapter/materialized_view_tests/_mixin.py @@ -11,6 +11,7 @@ run_dbt, set_model_file, ) + from tests.functional.adapter.materialized_view_tests import _files @@ -44,26 +45,8 @@ def my_other_base_table(self, project) -> BaseRelation: type=RelationType.Table, ) - @pytest.fixture(scope="class", autouse=True) - def seeds(self): - return {"my_seed.csv": _files.MY_SEED} - - @pytest.fixture(scope="class", autouse=True) - def models(self): - yield { - "my_table.sql": MY_TABLE, - "my_view.sql": MY_VIEW, - "my_base_table.sql": _files.MY_BASE_TABLE, - "my_other_base_table.sql": _files.MY_OTHER_BASE_TABLE, - "my_materialized_view.sql": _files.MY_MATERIALIZED_VIEW, - } - @pytest.fixture(scope="function", autouse=True) def setup(self, project, my_materialized_view): - """ - We overwrite this so that we get a `--full-refresh` of everything between tests, instead of just the MV - """ - # make sure the model in the data reflects the files each time run_dbt(["seed"]) run_dbt(["run", "--full-refresh"]) @@ -74,10 +57,22 @@ def setup(self, project, my_materialized_view): # and then reset them after the test runs set_model_file(project, my_materialized_view, initial_model) - - # ensure clean slate each method project.run_sql(f"drop schema if exists {project.test_schema} cascade") + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": _files.MY_SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_table.sql": MY_TABLE, + "my_view.sql": MY_VIEW, + "my_base_table.sql": _files.MY_BASE_TABLE, + "my_other_base_table.sql": _files.MY_OTHER_BASE_TABLE, + "my_materialized_view.sql": _files.MY_MATERIALIZED_VIEW, + } + @staticmethod def insert_record(project, table: BaseRelation, record: Tuple[int, int]) -> None: my_id, value = record From fae9e4fbb539cc8f36cbc0c502d9573168abba44 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 17 Jun 2024 20:43:17 -0400 Subject: [PATCH 15/22] pass in the new options config as context instead of the diffs --- dbt/adapters/bigquery/relation.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 38065dff6..70cd9e079 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -90,11 +90,24 @@ def materialized_view_config_changeset( new_materialized_view = cls.materialized_view_from_relation_config(relation_config) if new_materialized_view.options != existing_materialized_view.options: - # get an options change object with only the options that have changed - config_change_collection.options = BigQueryOptionsConfigChange.from_options_configs( - new_materialized_view.options, - existing_materialized_view.options, - ) + # the options have already gone through validation and defaults since they are on the MV object + # don't worry about things like interactions and defaults + + # allow_non_incremental_definition cannot be changed via an ALTER statement + if ( + new_materialized_view.options.allow_non_incremental_definition + != existing_materialized_view.options.allow_non_incremental_definition + ): + action = RelationConfigChangeAction.drop + else: + action = RelationConfigChangeAction.alter + + # even though some options are not changing, the change is issued as one DDL statement + # it is easier to submit a large set of consistent changes than determining a + # proper subset of consistent changes + context = new_materialized_view.options + + config_change_collection.options = BigQueryOptionsConfigChange(action, context) if new_materialized_view.partition != existing_materialized_view.partition: # the existing PartitionConfig is not hashable, but since we need to do From 79b8830c493f0fcb2cbc5355afc5d3f8eb9c8155 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 17 Jun 2024 20:44:32 -0400 Subject: [PATCH 16/22] add validation rules for inputs --- .../bigquery/relation_configs/_options.py | 89 +++++++++++++------ 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index 144c49e9f..8a8c9d6b7 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -1,22 +1,36 @@ -from dataclasses import asdict, dataclass +from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Any, Dict, Optional - -from dbt.adapters.relation_configs import RelationConfigChange +from typing import Any, Dict, Optional, Set + +from dbt.adapters.relation_configs import ( + RelationConfigChange, + RelationConfigChangeAction, + RelationConfigValidationMixin, + RelationConfigValidationRule, +) from dbt.adapters.contracts.relation import RelationConfig +from dbt_common.exceptions import DbtRuntimeError from google.cloud.bigquery import Table as BigQueryTable from typing_extensions import Self from dbt.adapters.bigquery.relation_configs._base import BigQueryBaseRelationConfig from dbt.adapters.bigquery.utility import bool_setting, float_setting, sql_escape -from dbt.adapters.relation_configs import RelationConfigChangeAction @dataclass(frozen=True, eq=True, unsafe_hash=True) -class BigQueryOptionsConfig(BigQueryBaseRelationConfig): +class BigQueryOptionsConfig(BigQueryBaseRelationConfig, RelationConfigValidationMixin): """ This config manages materialized view options. See the following for more information: - https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#materialized_view_option_list + + Note: + BigQuery allows options to be "unset" in the sense that they do not contain a value (think `None` or `null`). + This can be counterintuitive when that option is a boolean; it introduces a third value, in particular + a value that behaves "false-y". The practice is to mimic the data platform's inputs to the extent + possible to minimize any translation confusion between dbt docs and the platform's (BQ's) docs. + The values `False` and `None` will behave differently when producing the DDL options: + - `False` will show up in the statement submitted to BQ with the value `False` + - `None` will not show up in the statement submitted to BQ at all """ enable_refresh: Optional[bool] = True @@ -28,9 +42,38 @@ class BigQueryOptionsConfig(BigQueryBaseRelationConfig): description: Optional[str] = None labels: Optional[Dict[str, str]] = None - def as_ddl_dict(self) -> Dict[str, Any]: + @property + def validation_rules(self) -> Set[RelationConfigValidationRule]: + # validation_check is what is allowed + return { + RelationConfigValidationRule( + validation_check=self.allow_non_incremental_definition is not True + or self.max_staleness is not None, + validation_error=DbtRuntimeError( + "Please provide a setting for max_staleness when enabling allow_non_incremental_definition.\n" + "Received:\n" + f" allow_non_incremental_definition: {self.allow_non_incremental_definition}\n" + f" max_staleness: {self.max_staleness}\n" + ), + ), + RelationConfigValidationRule( + validation_check=self.enable_refresh is True + or all( + [self.max_staleness is None, self.allow_non_incremental_definition is None] + ), + validation_error=DbtRuntimeError( + "Do not provide a setting for refresh_interval_minutes, max_staleness, nor allow_non_incremental_definition when disabling enable_refresh.\n" + "Received:\n" + f" enable_refresh: {self.enable_refresh}\n" + f" max_staleness: {self.max_staleness}\n" + f" allow_non_incremental_definition: {self.allow_non_incremental_definition}\n" + ), + ), + } + + def as_ddl_dict(self, include_nulls: Optional[bool] = False) -> Dict[str, Any]: """ - Reformat `options_dict` so that it can be passed into the `bigquery_options()` macro. + Return a representation of this object so that it can be passed into the `bigquery_options()` macro. Options should be flattened and filtered prior to passing into this method. For example: - the "auto refresh" set of options should be flattened into the root instead of stuck under "auto_refresh" @@ -68,14 +111,17 @@ def array(x): def formatted_option(name: str) -> Optional[Any]: value = getattr(self, name) - if value is not None: - formatter = option_formatters[name] - return formatter(value) - return None + if value is None and include_nulls: + # used when altering relations to catch scenarios where non-defaulted options are "unset" + return "NULL" + elif value is None: + return None + formatter = option_formatters[name] + return formatter(value) options = { option: formatted_option(option) - for option, option_formatter in option_formatters.items() + for option in option_formatters if formatted_option(option) is not None } @@ -176,19 +222,4 @@ class BigQueryOptionsConfigChange(RelationConfigChange): @property def requires_full_refresh(self) -> bool: - # allow_non_incremental_definition cannot be changed via an ALTER statement - return self.context.allow_non_incremental_definition is not None - - @classmethod - def from_options_configs( - cls, new_options: BigQueryOptionsConfig, existing_options: BigQueryOptionsConfig - ) -> Self: - new_options_dict = asdict(new_options) - existing_options_dict = asdict(existing_options) - option_diffs_dict = { - k: v - for k, v in new_options_dict.items() - if new_options_dict[k] != existing_options_dict[k] - } - option_diffs = BigQueryOptionsConfig.from_dict(option_diffs_dict) - return cls(action=RelationConfigChangeAction.alter, context=option_diffs) + return self.action != RelationConfigChangeAction.alter From 2841b148abc7ddfdd85527c7085e6c7a1be3cf5b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 17 Jun 2024 20:44:56 -0400 Subject: [PATCH 17/22] allow for unsetting options during diffs --- .../bigquery/macros/relations/materialized_view/alter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/bigquery/macros/relations/materialized_view/alter.sql b/dbt/include/bigquery/macros/relations/materialized_view/alter.sql index e71f869ae..10d83900b 100644 --- a/dbt/include/bigquery/macros/relations/materialized_view/alter.sql +++ b/dbt/include/bigquery/macros/relations/materialized_view/alter.sql @@ -12,7 +12,7 @@ {% else %} alter materialized view {{ relation }} - set {{ bigquery_options(configuration_changes.options.context.as_ddl_dict()) }} + set {{ bigquery_options(configuration_changes.options.context.as_ddl_dict(include_nulls=True)) }} {%- endif %} From 1a9db60e243c03a477bed867a352378f69723e69 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 17 Jun 2024 20:45:45 -0400 Subject: [PATCH 18/22] add tests for refresh config changes --- .../adapter/materialized_view_tests/_files.py | 38 +++- .../test_materialized_view_refresh_changes.py | 206 ++++++++++++++++++ 2 files changed, 239 insertions(+), 5 deletions(-) create mode 100644 tests/functional/adapter/materialized_view_tests/test_materialized_view_refresh_changes.py diff --git a/tests/functional/adapter/materialized_view_tests/_files.py b/tests/functional/adapter/materialized_view_tests/_files.py index aa6803219..09870e4aa 100644 --- a/tests/functional/adapter/materialized_view_tests/_files.py +++ b/tests/functional/adapter/materialized_view_tests/_files.py @@ -71,11 +71,39 @@ MY_MINIMAL_MATERIALIZED_VIEW = """ -{{ - config( - materialized = 'materialized_view', - ) -}} +{{ config( + materialized='materialized_view' +) }} select * from {{ ref('my_seed') }} """ + + +MY_MATERIALIZED_VIEW_NON_INCREMENTAL = """ +{{ config( + max_staleness="INTERVAL '0-0 0 0:45:0' YEAR TO SECOND", + allow_non_incremental_definition=True, + materialized='materialized_view' +) }} +select + id, + value, + record_valid_date +from {{ ref('my_base_table') }} +""" + + +MY_MATERIALIZED_VIEW_REFRESH_CONFIG = """ +{{ config( + enable_refresh=True, + refresh_interval_minutes=60, + max_staleness="INTERVAL '0-0 0 0:45:0' YEAR TO SECOND", + allow_non_incremental_definition=True, + materialized='materialized_view' +) }} +select + id, + value, + record_valid_date +from {{ ref('my_base_table') }} +""" diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view_refresh_changes.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view_refresh_changes.py new file mode 100644 index 000000000..bef5931a8 --- /dev/null +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view_refresh_changes.py @@ -0,0 +1,206 @@ +from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.contracts.relation import RelationType +from dbt.tests.util import ( + get_connection, + get_model_file, + run_dbt, + set_model_file, +) +import pytest + +from tests.functional.adapter.materialized_view_tests import _files + + +class RefreshChanges: + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": _files.MY_SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_base_table.sql": _files.MY_BASE_TABLE, + "my_materialized_view.sql": _files.MY_MATERIALIZED_VIEW_INCREMENTAL, + } + + @pytest.fixture(scope="class") + def my_base_table(self, project) -> BaseRelation: + return project.adapter.Relation.create( + identifier="my_base_table", + schema=project.test_schema, + database=project.database, + type=RelationType.Table, + ) + + @pytest.fixture(scope="class") + def my_materialized_view(self, project) -> BaseRelation: + return project.adapter.Relation.create( + identifier="my_materialized_view", + schema=project.test_schema, + database=project.database, + type=RelationType.MaterializedView, + ) + + @staticmethod + def check_start_state(project, my_materialized_view): + raise NotImplementedError + + @staticmethod + def change_config(project, my_materialized_view): + raise NotImplementedError + + @staticmethod + def check_end_state(project, my_materialized_view): + raise NotImplementedError + + def test_refresh_changes(self, project, my_materialized_view): + # arrange + run_dbt(["seed"]) + run_dbt(["run"]) + self.check_start_state(project, my_materialized_view) + + # act + self.change_config(project, my_materialized_view) + run_dbt(["run", "--models", my_materialized_view.identifier]) + + # assert + self.check_end_state(project, my_materialized_view) + + +class TestTurnOnAllowNonIncremental(RefreshChanges): + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_base_table.sql": _files.MY_BASE_TABLE, + "my_materialized_view.sql": _files.MY_MINIMAL_MATERIALIZED_VIEW, + } + + @staticmethod + def check_start_state(project, my_materialized_view): + with get_connection(project.adapter): + results = project.adapter.describe_relation(my_materialized_view) + assert results.options.enable_refresh is True + assert results.options.refresh_interval_minutes == 30 + assert results.options.max_staleness is None + assert results.options.allow_non_incremental_definition is None + + @staticmethod + def change_config(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace( + "materialized='materialized_view'", + """materialized='materialized_view', allow_non_incremental_definition=True, max_staleness="INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" """, + ) + set_model_file(project, my_materialized_view, new_model) + + @staticmethod + def check_end_state(project, my_materialized_view): + with get_connection(project.adapter): + results = project.adapter.describe_relation(my_materialized_view) + assert results.options.enable_refresh is True + assert results.options.refresh_interval_minutes == 30 + assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.options.allow_non_incremental_definition is True + + +class TestTurnOffAllowNonIncremental(RefreshChanges): + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_base_table.sql": _files.MY_BASE_TABLE, + "my_materialized_view.sql": _files.MY_MATERIALIZED_VIEW_NON_INCREMENTAL, + } + + @staticmethod + def check_start_state(project, my_materialized_view): + with get_connection(project.adapter): + results = project.adapter.describe_relation(my_materialized_view) + assert results.options.enable_refresh is True + assert results.options.refresh_interval_minutes == 30 + assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.options.allow_non_incremental_definition is True + + @staticmethod + def change_config(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace( + "allow_non_incremental_definition=True", "allow_non_incremental_definition=False" + ) + set_model_file(project, my_materialized_view, new_model) + + @staticmethod + def check_end_state(project, my_materialized_view): + with get_connection(project.adapter): + results = project.adapter.describe_relation(my_materialized_view) + assert results.options.enable_refresh is True + assert results.options.refresh_interval_minutes == 30 + assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.options.allow_non_incremental_definition is False + + +class TestUnsetAllowNonIncremental(RefreshChanges): + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_base_table.sql": _files.MY_BASE_TABLE, + "my_materialized_view.sql": _files.MY_MATERIALIZED_VIEW_NON_INCREMENTAL, + } + + @staticmethod + def check_start_state(project, my_materialized_view): + with get_connection(project.adapter): + results = project.adapter.describe_relation(my_materialized_view) + assert results.options.enable_refresh is True + assert results.options.refresh_interval_minutes == 30 + assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.options.allow_non_incremental_definition is True + + @staticmethod + def change_config(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("allow_non_incremental_definition=True,", "") + set_model_file(project, my_materialized_view, new_model) + + @staticmethod + def check_end_state(project, my_materialized_view): + with get_connection(project.adapter): + results = project.adapter.describe_relation(my_materialized_view) + assert results.options.enable_refresh is True + assert results.options.refresh_interval_minutes == 30 + assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.options.allow_non_incremental_definition is None + + +class TestTurnOffRefresh(RefreshChanges): + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_base_table.sql": _files.MY_BASE_TABLE, + "my_materialized_view.sql": _files.MY_MATERIALIZED_VIEW_REFRESH_CONFIG, + } + + @staticmethod + def check_start_state(project, my_materialized_view): + with get_connection(project.adapter): + results = project.adapter.describe_relation(my_materialized_view) + assert results.options.enable_refresh is True + assert results.options.refresh_interval_minutes == 60 + assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.options.allow_non_incremental_definition is True + + @staticmethod + def change_config(project, my_materialized_view): + initial_model = get_model_file(project, my_materialized_view) + new_model = initial_model.replace("enable_refresh=True", "enable_refresh=False") + set_model_file(project, my_materialized_view, new_model) + + @staticmethod + def check_end_state(project, my_materialized_view): + with get_connection(project.adapter): + results = project.adapter.describe_relation(my_materialized_view) + assert results.options.enable_refresh is False + assert ( + results.options.refresh_interval_minutes == 30 + ) # this is a defaulted value in BQ, so it doesn't get cleared + assert results.options.max_staleness is None + assert results.options.allow_non_incremental_definition is None From 06019c92bfc19ce46f026d4b8933f19335d9fc41 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 20 Jun 2024 13:37:39 -0400 Subject: [PATCH 19/22] refactor materialized views to properly manage options changes --- dbt/adapters/bigquery/relation.py | 97 +++++--- .../bigquery/relation_configs/__init__.py | 9 +- .../bigquery/relation_configs/_base.py | 11 +- .../relation_configs/_materialized_view.py | 214 +++++++++++++---- .../bigquery/relation_configs/_options.py | 225 ------------------ .../bigquery/relation_configs/_partition.py | 3 +- .../relations/materialized_view/alter.sql | 59 ++++- .../relations/materialized_view/create.sql | 55 ++++- .../relations/materialized_view/replace.sql | 37 ++- .../adapter/materialized_view_tests/_files.py | 11 +- 10 files changed, 389 insertions(+), 332 deletions(-) delete mode 100644 dbt/adapters/bigquery/relation_configs/_options.py diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 70cd9e079..19dab336e 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -1,19 +1,18 @@ from dataclasses import dataclass, field +from itertools import chain, islice from typing import FrozenSet, Optional, TypeVar -from itertools import chain, islice from dbt.adapters.base.relation import BaseRelation, ComponentName, InformationSchema +from dbt.adapters.contracts.relation import RelationType, RelationConfig from dbt.adapters.relation_configs import RelationConfigChangeAction +from dbt_common.exceptions import CompilationError +from dbt_common.utils.dict import filter_null_values + from dbt.adapters.bigquery.relation_configs import ( - BigQueryClusterConfigChange, BigQueryMaterializedViewConfig, BigQueryMaterializedViewConfigChangeset, - BigQueryOptionsConfigChange, - BigQueryPartitionConfigChange, + BigQueryRelationConfigChange, ) -from dbt.adapters.contracts.relation import RelationType, RelationConfig -from dbt_common.exceptions import CompilationError -from dbt_common.utils.dict import filter_null_values Self = TypeVar("Self", bound="BigQueryRelation") @@ -89,43 +88,67 @@ def materialized_view_config_changeset( config_change_collection = BigQueryMaterializedViewConfigChangeset() new_materialized_view = cls.materialized_view_from_relation_config(relation_config) - if new_materialized_view.options != existing_materialized_view.options: - # the options have already gone through validation and defaults since they are on the MV object - # don't worry about things like interactions and defaults - - # allow_non_incremental_definition cannot be changed via an ALTER statement - if ( - new_materialized_view.options.allow_non_incremental_definition - != existing_materialized_view.options.allow_non_incremental_definition - ): - action = RelationConfigChangeAction.drop - else: - action = RelationConfigChangeAction.alter - - # even though some options are not changing, the change is issued as one DDL statement - # it is easier to submit a large set of consistent changes than determining a - # proper subset of consistent changes - context = new_materialized_view.options - - config_change_collection.options = BigQueryOptionsConfigChange(action, context) - - if new_materialized_view.partition != existing_materialized_view.partition: - # the existing PartitionConfig is not hashable, but since we need to do - # a full refresh either way, we don't need to provide a context - config_change_collection.partition = BigQueryPartitionConfigChange( - action=RelationConfigChangeAction.alter, + def add_change(option: str, requires_full_refresh: bool): + cls._add_change( + config_change_collection=config_change_collection, + new_relation=new_materialized_view, + existing_relation=existing_materialized_view, + option=option, + requires_full_refresh=requires_full_refresh, ) - if new_materialized_view.cluster != existing_materialized_view.cluster: - config_change_collection.cluster = BigQueryClusterConfigChange( - action=RelationConfigChangeAction.alter, - context=new_materialized_view.cluster, - ) + add_change("partition", True) + add_change("cluster", True) + add_change("enable_refresh", False) + add_change("refresh_interval_minutes", False) + add_change("max_staleness", False) + add_change("allow_non_incremental_definition", True) + add_change("kms_key_name", False) + add_change("description", False) + add_change("labels", False) if config_change_collection.has_changes: return config_change_collection return None + @classmethod + def _add_change( + cls, + config_change_collection, + new_relation, + existing_relation, + option: str, + requires_full_refresh: bool, + ) -> None: + # if there's no change, don't do anything + if getattr(new_relation, option) == getattr(existing_relation, option): + return + + # determine the type of change: drop, create, alter (includes full refresh) + if getattr(new_relation, option) is None: + action = RelationConfigChangeAction.drop + elif getattr(existing_relation, option) is None: + action = RelationConfigChangeAction.create + else: + action = RelationConfigChangeAction.alter + + # don't worry about passing along the context if it's a going to result in a full refresh + if requires_full_refresh: + context = None + else: + context = getattr(new_relation, option) + + # add the change to the collection for downstream processing + setattr( + config_change_collection, + option, + BigQueryRelationConfigChange( + action=action, + context=context, + requires_full_refresh=requires_full_refresh, + ), + ) + def information_schema(self, identifier: Optional[str] = None) -> "BigQueryInformationSchema": return BigQueryInformationSchema.from_relation(self, identifier) diff --git a/dbt/adapters/bigquery/relation_configs/__init__.py b/dbt/adapters/bigquery/relation_configs/__init__.py index 9ccdec1e0..8b76bbd32 100644 --- a/dbt/adapters/bigquery/relation_configs/__init__.py +++ b/dbt/adapters/bigquery/relation_configs/__init__.py @@ -1,4 +1,7 @@ -from dbt.adapters.bigquery.relation_configs._base import BigQueryBaseRelationConfig +from dbt.adapters.bigquery.relation_configs._base import ( + BigQueryBaseRelationConfig, + BigQueryRelationConfigChange, +) from dbt.adapters.bigquery.relation_configs._cluster import ( BigQueryClusterConfig, BigQueryClusterConfigChange, @@ -7,10 +10,6 @@ BigQueryMaterializedViewConfig, BigQueryMaterializedViewConfigChangeset, ) -from dbt.adapters.bigquery.relation_configs._options import ( - BigQueryOptionsConfig, - BigQueryOptionsConfigChange, -) from dbt.adapters.bigquery.relation_configs._partition import ( PartitionConfig, BigQueryPartitionConfigChange, diff --git a/dbt/adapters/bigquery/relation_configs/_base.py b/dbt/adapters/bigquery/relation_configs/_base.py index 45e29b99f..67e332d30 100644 --- a/dbt/adapters/bigquery/relation_configs/_base.py +++ b/dbt/adapters/bigquery/relation_configs/_base.py @@ -1,8 +1,8 @@ from dataclasses import dataclass -from typing import Optional, Dict, TYPE_CHECKING +from typing import Dict, Hashable, Optional, TYPE_CHECKING from dbt.adapters.base.relation import Policy -from dbt.adapters.relation_configs import RelationConfigBase +from dbt.adapters.relation_configs import RelationConfigBase, RelationConfigChangeAction from google.cloud.bigquery import Table as BigQueryTable from typing_extensions import Self @@ -66,3 +66,10 @@ def _get_first_row(cls, results: "agate.Table") -> "agate.Row": import agate return agate.Row(values=set()) + + +@dataclass(frozen=True, eq=True, unsafe_hash=True) +class BigQueryRelationConfigChange(RelationConfigBase): + action: RelationConfigChangeAction + context: Optional[Hashable] + requires_full_refresh: Optional[bool] = False diff --git a/dbt/adapters/bigquery/relation_configs/_materialized_view.py b/dbt/adapters/bigquery/relation_configs/_materialized_view.py index 81ca6b3de..442d31196 100644 --- a/dbt/adapters/bigquery/relation_configs/_materialized_view.py +++ b/dbt/adapters/bigquery/relation_configs/_materialized_view.py @@ -1,32 +1,34 @@ -from dataclasses import dataclass -from typing import Any, Dict, Optional +from dataclasses import dataclass, fields +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional, Set from dbt.adapters.contracts.relation import ( - RelationConfig, ComponentName, + RelationConfig, +) +from dbt.adapters.relation_configs import ( + RelationConfigValidationMixin, + RelationConfigValidationRule, ) +from dbt_common.exceptions import DbtRuntimeError from google.cloud.bigquery import Table as BigQueryTable +from typing_extensions import Self -from dbt.adapters.bigquery.relation_configs._base import BigQueryBaseRelationConfig -from dbt.adapters.bigquery.relation_configs._options import ( - BigQueryOptionsConfig, - BigQueryOptionsConfigChange, -) -from dbt.adapters.bigquery.relation_configs._partition import ( - BigQueryPartitionConfigChange, - PartitionConfig, -) -from dbt.adapters.bigquery.relation_configs._cluster import ( - BigQueryClusterConfig, - BigQueryClusterConfigChange, +from dbt.adapters.bigquery.relation_configs._base import ( + BigQueryBaseRelationConfig, + BigQueryRelationConfigChange, ) +from dbt.adapters.bigquery.relation_configs._cluster import BigQueryClusterConfig +from dbt.adapters.bigquery.relation_configs._partition import PartitionConfig +from dbt.adapters.bigquery.utility import bool_setting, float_setting @dataclass(frozen=True, eq=True, unsafe_hash=True) -class BigQueryMaterializedViewConfig(BigQueryBaseRelationConfig): +class BigQueryMaterializedViewConfig(BigQueryBaseRelationConfig, RelationConfigValidationMixin): """ This config follow the specs found here: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_materialized_view_statement + https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#materialized_view_option_list The following parameters are configurable by dbt: - table_id: name of the materialized view @@ -35,64 +37,159 @@ class BigQueryMaterializedViewConfig(BigQueryBaseRelationConfig): - options: options that get set in `SET OPTIONS()` clause - partition: object containing partition information - cluster: object containing cluster information + + The following options are configurable by dbt: + - enable_refresh: turns on/off automatic refresh + - refresh_interval_minutes: the refresh interval in minutes, when enable_refresh is True + - expiration_timestamp: the expiration of data in the underlying table + - max_staleness: the oldest data can be before triggering a refresh + - allow_non_incremental_definition: allows non-incremental reloads, requires max_staleness + - kms_key_name: the name of the keyfile + - description: the comment to add to the materialized view + - labels: labels to add to the materialized view + + Note: + BigQuery allows options to be "unset" in the sense that they do not contain a value (think `None` or `null`). + This can be counterintuitive when that option is a boolean; it introduces a third value, in particular + a value that behaves "false-y". The practice is to mimic the data platform's inputs to the extent + possible to minimize any translation confusion between dbt docs and the platform's (BQ's) docs. + The values `False` and `None` will behave differently when producing the DDL options: + - `False` will show up in the statement submitted to BQ with the value `False` + - `None` will not show up in the statement submitted to BQ at all """ table_id: str dataset_id: str project_id: str - options: BigQueryOptionsConfig partition: Optional[PartitionConfig] = None cluster: Optional[BigQueryClusterConfig] = None + enable_refresh: Optional[bool] = True + refresh_interval_minutes: Optional[float] = 30 + expiration_timestamp: Optional[datetime] = None + max_staleness: Optional[str] = None + allow_non_incremental_definition: Optional[bool] = None + kms_key_name: Optional[str] = None + description: Optional[str] = None + labels: Optional[Dict[str, str]] = None + + @property + def validation_rules(self) -> Set[RelationConfigValidationRule]: + # validation_check is what is allowed + return { + RelationConfigValidationRule( + validation_check=self.allow_non_incremental_definition is not True + or self.max_staleness is not None, + validation_error=DbtRuntimeError( + "Please provide a setting for max_staleness when enabling allow_non_incremental_definition.\n" + "Received:\n" + f" allow_non_incremental_definition: {self.allow_non_incremental_definition}\n" + f" max_staleness: {self.max_staleness}\n" + ), + ), + RelationConfigValidationRule( + validation_check=self.enable_refresh is True + or all( + [self.max_staleness is None, self.allow_non_incremental_definition is None] + ), + validation_error=DbtRuntimeError( + "Do not provide a setting for refresh_interval_minutes, max_staleness, nor allow_non_incremental_definition when disabling enable_refresh.\n" + "Received:\n" + f" enable_refresh: {self.enable_refresh}\n" + f" max_staleness: {self.max_staleness}\n" + f" allow_non_incremental_definition: {self.allow_non_incremental_definition}\n" + ), + ), + } @classmethod - def from_dict(cls, config_dict: Dict[str, Any]) -> "BigQueryMaterializedViewConfig": + def from_dict(cls, config_dict: Dict[str, Any]) -> Self: # required kwargs_dict: Dict[str, Any] = { "table_id": cls._render_part(ComponentName.Identifier, config_dict["table_id"]), "dataset_id": cls._render_part(ComponentName.Schema, config_dict["dataset_id"]), "project_id": cls._render_part(ComponentName.Database, config_dict["project_id"]), - "options": BigQueryOptionsConfig.from_dict(config_dict["options"]), } # optional - if partition := config_dict.get("partition"): - kwargs_dict.update({"partition": PartitionConfig.parse(partition)}) + optional_settings = { + "partition": PartitionConfig.parse, + "cluster": BigQueryClusterConfig.from_dict, + "enable_refresh": bool_setting, + "refresh_interval_minutes": float_setting, + "expiration_timestamp": None, + "max_staleness": None, + "allow_non_incremental_definition": bool_setting, + "kms_key_name": None, + "description": None, + "labels": None, + } - if cluster := config_dict.get("cluster"): - kwargs_dict.update({"cluster": BigQueryClusterConfig.from_dict(cluster)}) + for setting, parser in optional_settings.items(): + value = config_dict.get(setting) + if value is not None and parser is not None: + kwargs_dict.update({setting: parser(value)}) # type: ignore + elif value is not None: + kwargs_dict.update({setting: value}) - materialized_view: "BigQueryMaterializedViewConfig" = super().from_dict(kwargs_dict) # type: ignore + materialized_view: Self = super().from_dict(kwargs_dict) # type: ignore return materialized_view @classmethod def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]: - config_dict = { + config_extras = relation_config.config.extra # type: ignore + + config_dict: Dict[str, Any] = { + # required "table_id": relation_config.identifier, "dataset_id": relation_config.schema, "project_id": relation_config.database, - # despite this being a foreign object, there will always be options because of defaults - "options": BigQueryOptionsConfig.parse_relation_config(relation_config), + # optional - no transformations + "enable_refresh": config_extras.get("enable_refresh"), + "refresh_interval_minutes": config_extras.get("refresh_interval_minutes"), + "max_staleness": config_extras.get("max_staleness"), + "allow_non_incremental_definition": config_extras.get( + "allow_non_incremental_definition" + ), + "kms_key_name": config_extras.get("kms_key_name"), + "description": config_extras.get("description"), + "labels": config_extras.get("labels"), } - # optional - if relation_config.config and "partition_by" in relation_config.config: - config_dict.update({"partition": PartitionConfig.parse_model_node(relation_config)}) + # optional - transformations + if relation_config.config.get("partition_by"): # type: ignore + config_dict["partition"] = PartitionConfig.parse_model_node(relation_config) + + if relation_config.config.get("cluster_by"): # type: ignore + config_dict["cluster"] = BigQueryClusterConfig.parse_relation_config(relation_config) - if relation_config.config and "cluster_by" in relation_config.config: - config_dict.update( - {"cluster": BigQueryClusterConfig.parse_relation_config(relation_config)} + if hours_to_expiration := config_extras.get("hours_to_expiration"): + config_dict["expiration_timestamp"] = datetime.now(tz=timezone.utc) + timedelta( + hours=hours_to_expiration ) + if relation_config.config.persist_docs and config_extras.get("description"): # type: ignore + config_dict["description"] = config_extras.get("description") + return config_dict @classmethod def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: - config_dict = { + config_dict: Dict[str, Any] = { + # required "table_id": table.table_id, "dataset_id": table.dataset_id, "project_id": table.project, - # despite this being a foreign object, there will always be options because of defaults - "options": BigQueryOptionsConfig.parse_bq_table(table), + # optional - no transformation + "enable_refresh": table.mview_enable_refresh, + "expiration_timestamp": table.expires, + "allow_non_incremental_definition": table._properties.get("materializedView", {}).get( + "allowNonIncrementalDefinition" + ), + "kms_key_name": getattr( + getattr(table, "encryption_configuration"), "kms_key_name", None + ), + "description": table.description, + "labels": table.labels if table.labels != {} else None, } # optional @@ -102,31 +199,46 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: if table.clustering_fields: config_dict.update({"cluster": BigQueryClusterConfig.parse_bq_table(table)}) + if refresh_interval_seconds := table.mview_refresh_interval.seconds: + config_dict.update({"refresh_interval_minutes": refresh_interval_seconds / 60}) + + if max_staleness := table._properties.get("maxStaleness"): + config_dict.update({"max_staleness": f"INTERVAL '{max_staleness}' YEAR TO SECOND"}) + return config_dict @dataclass class BigQueryMaterializedViewConfigChangeset: - options: Optional[BigQueryOptionsConfigChange] = None - partition: Optional[BigQueryPartitionConfigChange] = None - cluster: Optional[BigQueryClusterConfigChange] = None + """ + A collection of changes on a materialized view. + + Note: We don't watch for `expiration_timestamp` because it only gets set on the initial creation. + It would naturally change every time since it's set via `hours_to_expiration`, which would push out + the calculated `expiration_timestamp`. + """ + + partition: Optional[BigQueryRelationConfigChange] = None + cluster: Optional[BigQueryRelationConfigChange] = None + enable_refresh: Optional[BigQueryRelationConfigChange] = None + refresh_interval_minutes: Optional[BigQueryRelationConfigChange] = None + max_staleness: Optional[BigQueryRelationConfigChange] = None + allow_non_incremental_definition: Optional[BigQueryRelationConfigChange] = None + kms_key_name: Optional[BigQueryRelationConfigChange] = None + description: Optional[BigQueryRelationConfigChange] = None + labels: Optional[BigQueryRelationConfigChange] = None @property def requires_full_refresh(self) -> bool: return any( - { - self.options.requires_full_refresh if self.options else False, - self.partition.requires_full_refresh if self.partition else False, - self.cluster.requires_full_refresh if self.cluster else False, - } + [ + getattr(self, field.name).requires_full_refresh + if getattr(self, field.name) + else False + for field in fields(self) + ] ) @property def has_changes(self) -> bool: - return any( - { - self.options if self.options else False, - self.partition if self.partition else False, - self.cluster if self.cluster else False, - } - ) + return any([getattr(self, field.name) is not None for field in fields(self)]) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py deleted file mode 100644 index 8a8c9d6b7..000000000 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ /dev/null @@ -1,225 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime, timedelta -from typing import Any, Dict, Optional, Set - -from dbt.adapters.relation_configs import ( - RelationConfigChange, - RelationConfigChangeAction, - RelationConfigValidationMixin, - RelationConfigValidationRule, -) -from dbt.adapters.contracts.relation import RelationConfig -from dbt_common.exceptions import DbtRuntimeError -from google.cloud.bigquery import Table as BigQueryTable -from typing_extensions import Self - -from dbt.adapters.bigquery.relation_configs._base import BigQueryBaseRelationConfig -from dbt.adapters.bigquery.utility import bool_setting, float_setting, sql_escape - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class BigQueryOptionsConfig(BigQueryBaseRelationConfig, RelationConfigValidationMixin): - """ - This config manages materialized view options. See the following for more information: - - https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#materialized_view_option_list - - Note: - BigQuery allows options to be "unset" in the sense that they do not contain a value (think `None` or `null`). - This can be counterintuitive when that option is a boolean; it introduces a third value, in particular - a value that behaves "false-y". The practice is to mimic the data platform's inputs to the extent - possible to minimize any translation confusion between dbt docs and the platform's (BQ's) docs. - The values `False` and `None` will behave differently when producing the DDL options: - - `False` will show up in the statement submitted to BQ with the value `False` - - `None` will not show up in the statement submitted to BQ at all - """ - - enable_refresh: Optional[bool] = True - refresh_interval_minutes: Optional[float] = 30 - expiration_timestamp: Optional[datetime] = None - max_staleness: Optional[str] = None - allow_non_incremental_definition: Optional[bool] = None - kms_key_name: Optional[str] = None - description: Optional[str] = None - labels: Optional[Dict[str, str]] = None - - @property - def validation_rules(self) -> Set[RelationConfigValidationRule]: - # validation_check is what is allowed - return { - RelationConfigValidationRule( - validation_check=self.allow_non_incremental_definition is not True - or self.max_staleness is not None, - validation_error=DbtRuntimeError( - "Please provide a setting for max_staleness when enabling allow_non_incremental_definition.\n" - "Received:\n" - f" allow_non_incremental_definition: {self.allow_non_incremental_definition}\n" - f" max_staleness: {self.max_staleness}\n" - ), - ), - RelationConfigValidationRule( - validation_check=self.enable_refresh is True - or all( - [self.max_staleness is None, self.allow_non_incremental_definition is None] - ), - validation_error=DbtRuntimeError( - "Do not provide a setting for refresh_interval_minutes, max_staleness, nor allow_non_incremental_definition when disabling enable_refresh.\n" - "Received:\n" - f" enable_refresh: {self.enable_refresh}\n" - f" max_staleness: {self.max_staleness}\n" - f" allow_non_incremental_definition: {self.allow_non_incremental_definition}\n" - ), - ), - } - - def as_ddl_dict(self, include_nulls: Optional[bool] = False) -> Dict[str, Any]: - """ - Return a representation of this object so that it can be passed into the `bigquery_options()` macro. - - Options should be flattened and filtered prior to passing into this method. For example: - - the "auto refresh" set of options should be flattened into the root instead of stuck under "auto_refresh" - - any option that comes in set as `None` will be unset; this happens mostly due to config changes - """ - - def boolean(x): - return x - - def numeric(x): - return x - - def string(x): - return f"'{x}'" - - def escaped_string(x): - return f'"""{sql_escape(x)}"""' - - def interval(x): - return x - - def array(x): - return list(x.items()) - - option_formatters = { - "enable_refresh": boolean, - "refresh_interval_minutes": numeric, - "expiration_timestamp": interval, - "max_staleness": interval, - "allow_non_incremental_definition": boolean, - "kms_key_name": string, - "description": escaped_string, - "labels": array, - } - - def formatted_option(name: str) -> Optional[Any]: - value = getattr(self, name) - if value is None and include_nulls: - # used when altering relations to catch scenarios where non-defaulted options are "unset" - return "NULL" - elif value is None: - return None - formatter = option_formatters[name] - return formatter(value) - - options = { - option: formatted_option(option) - for option in option_formatters - if formatted_option(option) is not None - } - - return options - - @classmethod - def from_dict(cls, config_dict: Dict[str, Any]) -> Self: - setting_formatters = { - "enable_refresh": bool_setting, - "refresh_interval_minutes": float_setting, - "expiration_timestamp": None, - "max_staleness": None, - "allow_non_incremental_definition": bool_setting, - "kms_key_name": None, - "description": None, - "labels": None, - } - - def formatted_setting(name: str) -> Any: - value = config_dict.get(name) - if formatter := setting_formatters[name]: - return formatter(value) - return value - - kwargs_dict = {attribute: formatted_setting(attribute) for attribute in setting_formatters} - - # avoid picking up defaults on dependent options - # e.g. don't set `refresh_interval_minutes` = 30 when the user has `enable_refresh` = False - if kwargs_dict["enable_refresh"] is False: - kwargs_dict.update( - { - "refresh_interval_minutes": None, - "max_staleness": None, - "allow_non_incremental_definition": None, - } - ) - - options: Self = super().from_dict(kwargs_dict) # type: ignore - return options - - @classmethod - def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any]: - config_dict = { - option: relation_config.config.extra.get(option) # type: ignore - for option in [ - "enable_refresh", - "refresh_interval_minutes", - "expiration_timestamp", - "max_staleness", - "allow_non_incremental_definition", - "kms_key_name", - "description", - "labels", - ] - } - - # update dbt-specific versions of these settings - if hours_to_expiration := relation_config.config.extra.get( # type: ignore - "hours_to_expiration" - ): # type: ignore - config_dict.update( - {"expiration_timestamp": datetime.now() + timedelta(hours=hours_to_expiration)} - ) - if not relation_config.config.persist_docs: # type: ignore - del config_dict["description"] - - return config_dict - - @classmethod - def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: - config_dict = { - "enable_refresh": table.mview_enable_refresh, - "refresh_interval_minutes": table.mview_refresh_interval.seconds / 60, - "expiration_timestamp": table.expires, - "max_staleness": ( - f"INTERVAL '{table._properties.get('maxStaleness')}' YEAR TO SECOND" - if table._properties.get("maxStaleness") - else None - ), - "allow_non_incremental_definition": table._properties.get("materializedView", {}).get( - "allowNonIncrementalDefinition" - ), - "description": table.description, - } - - # map the empty dict to None - if labels := table.labels: - config_dict.update({"labels": labels}) - - if encryption_configuration := table.encryption_configuration: - config_dict.update({"kms_key_name": encryption_configuration.kms_key_name}) - return config_dict - - -@dataclass(frozen=True, eq=True, unsafe_hash=True) -class BigQueryOptionsConfigChange(RelationConfigChange): - context: BigQueryOptionsConfig - - @property - def requires_full_refresh(self) -> bool: - return self.action != RelationConfigChangeAction.alter diff --git a/dbt/adapters/bigquery/relation_configs/_partition.py b/dbt/adapters/bigquery/relation_configs/_partition.py index 555aa3664..3c24e129e 100644 --- a/dbt/adapters/bigquery/relation_configs/_partition.py +++ b/dbt/adapters/bigquery/relation_configs/_partition.py @@ -6,6 +6,7 @@ from dbt.adapters.contracts.relation import RelationConfig from dbt_common.dataclass_schema import dbtClassMixin, ValidationError from google.cloud.bigquery.table import Table as BigQueryTable +from typing_extensions import Self @dataclass @@ -80,7 +81,7 @@ def render_wrapped(self, alias: Optional[str] = None): return self.render(alias) @classmethod - def parse(cls, raw_partition_by) -> Optional["PartitionConfig"]: + def parse(cls, raw_partition_by: Dict[str, Any]) -> Optional[Self]: if raw_partition_by is None: return None try: diff --git a/dbt/include/bigquery/macros/relations/materialized_view/alter.sql b/dbt/include/bigquery/macros/relations/materialized_view/alter.sql index 10d83900b..b259f7983 100644 --- a/dbt/include/bigquery/macros/relations/materialized_view/alter.sql +++ b/dbt/include/bigquery/macros/relations/materialized_view/alter.sql @@ -11,8 +11,65 @@ {{ get_replace_sql(existing_relation, relation, sql) }} {% else %} + {%- set _needs_comma = False -%} + alter materialized view {{ relation }} - set {{ bigquery_options(configuration_changes.options.context.as_ddl_dict(include_nulls=True)) }} + set options( + {% if configuration_changes.enable_refresh -%} + {%- if _needs_comma -%},{%- endif -%} + {%- if configuration_changes.enable_refresh.action == 'drop' -%} + enable_refresh = NULL + {%- else -%} + enable_refresh = {{ configuration_changes.enable_refresh.context }} + {%- endif -%} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if configuration_changes.refresh_interval_minutes -%} + {%- if _needs_comma -%},{%- endif -%} + {%- if configuration_changes.refresh_interval_minutes.action == 'drop' -%} + refresh_interval_minutes = NULL + {%- else -%} + refresh_interval_minutes = {{ configuration_changes.refresh_interval_minutes.context }} + {%- endif -%} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if configuration_changes.max_staleness -%} + {%- if _needs_comma -%},{%- endif -%} + {%- if configuration_changes.max_staleness.action == 'drop' -%} + max_staleness = NULL + {%- else -%} + max_staleness = {{ configuration_changes.max_staleness.context }} + {%- endif -%} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if configuration_changes.kms_key_name -%} + {%- if _needs_comma -%},{%- endif -%} + {%- if configuration_changes.kms_key_name.action == 'drop' -%} + kms_key_name = NULL + {%- else -%} + kms_key_name = '{{ configuration_changes.kms_key_name.context }}' + {%- endif -%} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if configuration_changes.description -%} + {%- if _needs_comma -%},{%- endif -%} + {%- if configuration_changes.description.action == 'drop' -%} + description = NULL + {%- else -%} + description = ""{{ configuration_changes.description.context|tojson|safe }}"" + {%- endif -%} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if configuration_changes.labels -%} + {%- if _needs_comma -%},{%- endif -%} + {%- if configuration_changes.labels.action == 'drop' -%} + labels = NULL + {%- else -%} + labels = {{ configuration_changes.labels.context.items()|list }} + {%- endif -%} + {%- set _needs_comma = True -%} + {%- endif -%} + ) {%- endif %} diff --git a/dbt/include/bigquery/macros/relations/materialized_view/create.sql b/dbt/include/bigquery/macros/relations/materialized_view/create.sql index d3e8c7685..35ec08b4e 100644 --- a/dbt/include/bigquery/macros/relations/materialized_view/create.sql +++ b/dbt/include/bigquery/macros/relations/materialized_view/create.sql @@ -2,10 +2,57 @@ {%- set materialized_view = adapter.Relation.materialized_view_from_relation_config(config.model) -%} - create materialized view if not exists {{ relation }} - {% if materialized_view.partition %}{{ partition_by(materialized_view.partition) }}{% endif %} - {% if materialized_view.cluster %}{{ cluster_by(materialized_view.cluster.fields) }}{% endif %} - {{ bigquery_options(materialized_view.options.as_ddl_dict()) }} + {%- set _needs_comma = False -%} + + create materialized view {{ relation }} + {% if materialized_view.partition -%} + {{ partition_by(materialized_view.partition) }} + {% endif -%} + {%- if materialized_view.cluster -%} + {{ cluster_by(materialized_view.cluster.fields) }} + {%- endif %} + options( + {% if materialized_view.enable_refresh -%} + {%- if _needs_comma -%},{%- endif -%} + enable_refresh = {{ materialized_view.enable_refresh }} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if materialized_view.refresh_interval_minutes -%} + {%- if _needs_comma -%},{%- endif -%} + refresh_interval_minutes = {{ materialized_view.refresh_interval_minutes }} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if materialized_view.expiration_timestamp -%} + {%- if _needs_comma -%},{%- endif -%} + expiration_timestamp = '{{ materialized_view.expiration_timestamp }}' + {%- set _needs_comma = True -%} + {% endif -%} + {%- if materialized_view.max_staleness -%} + {%- if _needs_comma -%},{%- endif -%} + max_staleness = {{ materialized_view.max_staleness }} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if materialized_view.allow_non_incremental_definition -%} + {%- if _needs_comma -%},{%- endif -%} + allow_non_incremental_definition = {{ materialized_view.allow_non_incremental_definition }} + {%- set _needs_comma = True -%} + {% endif -%} + {%- if materialized_view.kms_key_name -%} + {%- if _needs_comma -%},{%- endif -%} + kms_key_name = '{{ materialized_view.kms_key_name }}' + {%- set _needs_comma = True -%} + {% endif -%} + {%- if materialized_view.description -%} + {%- if _needs_comma -%},{%- endif -%} + description = ""{{ materialized_view.description|tojson|safe }}"" + {%- set _needs_comma = True -%} + {% endif -%} + {%- if materialized_view.labels -%} + {%- if _needs_comma -%},{%- endif -%} + labels = {{ materialized_view.labels.items()|list }} + {%- set _needs_comma = True -%} + {%- endif %} + ) as {{ sql }} {% endmacro %} diff --git a/dbt/include/bigquery/macros/relations/materialized_view/replace.sql b/dbt/include/bigquery/macros/relations/materialized_view/replace.sql index 2e4a0b69f..40d9ad7c9 100644 --- a/dbt/include/bigquery/macros/relations/materialized_view/replace.sql +++ b/dbt/include/bigquery/macros/relations/materialized_view/replace.sql @@ -2,10 +2,39 @@ {%- set materialized_view = adapter.Relation.materialized_view_from_relation_config(config.model) -%} - create or replace materialized view if not exists {{ relation }} - {% if materialized_view.partition %}{{ partition_by(materialized_view.partition) }}{% endif %} - {% if materialized_view.cluster %}{{ cluster_by(materialized_view.cluster.fields) }}{% endif %} - {{ bigquery_options(materialized_view.options.as_ddl_dict()) }} + create or replace materialized view {{ relation }} + {% if materialized_view.partition -%} + {{ partition_by(materialized_view.partition) }} + {% endif -%} + {%- if materialized_view.cluster -%} + {{ cluster_by(materialized_view.cluster.fields) }} + {%- endif %} + options( + {% if materialized_view.enable_refresh -%} + enable_refresh = {{ materialized_view.enable_refresh }} + {% endif -%} + {%- if materialized_view.refresh_interval_minutes -%} + ,refresh_interval_minutes = {{ materialized_view.refresh_interval_minutes }} + {% endif -%} + {%- if materialized_view.expiration_timestamp -%} + ,expiration_timestamp = '{{ materialized_view.expiration_timestamp }}' + {% endif -%} + {%- if materialized_view.max_staleness -%} + ,max_staleness = {{ materialized_view.max_staleness }} + {% endif -%} + {%- if materialized_view.allow_non_incremental_definition -%} + ,allow_non_incremental_definition = {{ materialized_view.allow_non_incremental_definition }} + {% endif -%} + {%- if materialized_view.kms_key_name -%} + ,kms_key_name = '{{ materialized_view.kms_key_name }}' + {% endif -%} + {%- if materialized_view.description -%} + ,description = ""{{ materialized_view.description|tojson|safe }}"" + {% endif -%} + {%- if materialized_view.labels -%} + ,labels = {{ materialized_view.labels.items()|list }} + {%- endif %} + ) as {{ sql }} {% endmacro %} diff --git a/tests/functional/adapter/materialized_view_tests/_files.py b/tests/functional/adapter/materialized_view_tests/_files.py index 09870e4aa..5bd43a251 100644 --- a/tests/functional/adapter/materialized_view_tests/_files.py +++ b/tests/functional/adapter/materialized_view_tests/_files.py @@ -1,3 +1,6 @@ +# flake8: noqa +# ignores the special characters in the descripton check + MY_SEED = """ id,value,record_valid_date 1,100,2023-01-01 00:00:00 @@ -25,9 +28,10 @@ # the whitespace to the left on partition matters here +# this should test all possible config (skip KMS key since it needs to exist) MY_MATERIALIZED_VIEW = """ {{ config( - materialized='materialized_view', + materialized="materialized_view", partition_by={ "field": "record_valid_date", "data_type": "datetime", @@ -36,8 +40,11 @@ cluster_by=["id", "value"], enable_refresh=True, refresh_interval_minutes=60, + hours_to_expiration=24, max_staleness="INTERVAL '0-0 0 0:45:0' YEAR TO SECOND", - allow_non_incremental_definition=True + allow_non_incremental_definition=True, + description=" Date: Thu, 20 Jun 2024 13:38:15 -0400 Subject: [PATCH 20/22] changelog --- .changes/unreleased/Features-20231108-140752.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Features-20231108-140752.yaml b/.changes/unreleased/Features-20231108-140752.yaml index 2a9a41a07..56dcc6a60 100644 --- a/.changes/unreleased/Features-20231108-140752.yaml +++ b/.changes/unreleased/Features-20231108-140752.yaml @@ -2,5 +2,5 @@ kind: Features body: Support allow_non_incremental_definition option in BigQuery materialized views time: 2023-11-08T14:07:52.28972-05:00 custom: - Author: bnaul + Author: bnaul,mikealfare Issue: "672" From 05402b7e2ee1fb1710ac9cd4f247351a241c9885 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 20 Jun 2024 13:43:52 -0400 Subject: [PATCH 21/22] update replace to match create --- .../relations/materialized_view/replace.sql | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/dbt/include/bigquery/macros/relations/materialized_view/replace.sql b/dbt/include/bigquery/macros/relations/materialized_view/replace.sql index 40d9ad7c9..9357dc955 100644 --- a/dbt/include/bigquery/macros/relations/materialized_view/replace.sql +++ b/dbt/include/bigquery/macros/relations/materialized_view/replace.sql @@ -2,6 +2,8 @@ {%- set materialized_view = adapter.Relation.materialized_view_from_relation_config(config.model) -%} + {%- set _needs_comma = False -%} + create or replace materialized view {{ relation }} {% if materialized_view.partition -%} {{ partition_by(materialized_view.partition) }} @@ -11,28 +13,44 @@ {%- endif %} options( {% if materialized_view.enable_refresh -%} + {%- if _needs_comma -%},{%- endif -%} enable_refresh = {{ materialized_view.enable_refresh }} + {%- set _needs_comma = True -%} {% endif -%} {%- if materialized_view.refresh_interval_minutes -%} - ,refresh_interval_minutes = {{ materialized_view.refresh_interval_minutes }} + {%- if _needs_comma -%},{%- endif -%} + refresh_interval_minutes = {{ materialized_view.refresh_interval_minutes }} + {%- set _needs_comma = True -%} {% endif -%} {%- if materialized_view.expiration_timestamp -%} - ,expiration_timestamp = '{{ materialized_view.expiration_timestamp }}' + {%- if _needs_comma -%},{%- endif -%} + expiration_timestamp = '{{ materialized_view.expiration_timestamp }}' + {%- set _needs_comma = True -%} {% endif -%} {%- if materialized_view.max_staleness -%} - ,max_staleness = {{ materialized_view.max_staleness }} + {%- if _needs_comma -%},{%- endif -%} + max_staleness = {{ materialized_view.max_staleness }} + {%- set _needs_comma = True -%} {% endif -%} {%- if materialized_view.allow_non_incremental_definition -%} - ,allow_non_incremental_definition = {{ materialized_view.allow_non_incremental_definition }} + {%- if _needs_comma -%},{%- endif -%} + allow_non_incremental_definition = {{ materialized_view.allow_non_incremental_definition }} + {%- set _needs_comma = True -%} {% endif -%} {%- if materialized_view.kms_key_name -%} - ,kms_key_name = '{{ materialized_view.kms_key_name }}' + {%- if _needs_comma -%},{%- endif -%} + kms_key_name = '{{ materialized_view.kms_key_name }}' + {%- set _needs_comma = True -%} {% endif -%} {%- if materialized_view.description -%} - ,description = ""{{ materialized_view.description|tojson|safe }}"" + {%- if _needs_comma -%},{%- endif -%} + description = ""{{ materialized_view.description|tojson|safe }}"" + {%- set _needs_comma = True -%} {% endif -%} {%- if materialized_view.labels -%} - ,labels = {{ materialized_view.labels.items()|list }} + {%- if _needs_comma -%},{%- endif -%} + labels = {{ materialized_view.labels.items()|list }} + {%- set _needs_comma = True -%} {%- endif %} ) as {{ sql }} From 1f922408ac1b1cf988f114baa476e7ceddce6bf9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 20 Jun 2024 15:05:19 -0400 Subject: [PATCH 22/22] update tests to reflect removing the options subconfig --- .../test_materialized_view_changes.py | 10 +-- .../test_materialized_view_cluster_changes.py | 8 +-- ...est_materialized_view_partition_changes.py | 8 +-- .../test_materialized_view_refresh_changes.py | 64 +++++++++---------- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py index 1a94b93ec..e61ebf8c9 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view_changes.py @@ -17,10 +17,10 @@ def check_start_state(project, materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(materialized_view) assert isinstance(results, BigQueryMaterializedViewConfig) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 60 - assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" - assert results.options.allow_non_incremental_definition is True + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 60 + assert results.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.allow_non_incremental_definition is True assert results.partition.field == "record_valid_date" assert results.partition.data_type == "datetime" assert results.partition.granularity == "day" @@ -80,7 +80,7 @@ def check_state_replace_change_is_applied(project, materialized_view): assert results.partition.data_type == "int64" assert results.partition.range == {"start": 0, "end": 500, "interval": 50} assert results.cluster.fields == frozenset({"id"}) - assert results.options.allow_non_incremental_definition is False + assert results.allow_non_incremental_definition is False class TestBigQueryMaterializedViewChangesApply( diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view_cluster_changes.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view_cluster_changes.py index 74e174d4f..e7ef5d903 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view_cluster_changes.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view_cluster_changes.py @@ -19,8 +19,8 @@ def check_start_state(project, materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(materialized_view) assert isinstance(results, BigQueryMaterializedViewConfig) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 60 + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 60 assert results.cluster.fields == frozenset({"id", "value"}) @staticmethod @@ -34,8 +34,8 @@ def check_state_alter_change_is_applied(project, materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(materialized_view) assert isinstance(results, BigQueryMaterializedViewConfig) - assert results.options.enable_refresh is False - assert results.options.refresh_interval_minutes == 30 # BQ returns it to the default + assert results.enable_refresh is False + assert results.refresh_interval_minutes == 30 # BQ returns it to the default @staticmethod def change_config_via_replace(project, materialized_view): diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view_partition_changes.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view_partition_changes.py index 7f396ae1b..deaf3de45 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view_partition_changes.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view_partition_changes.py @@ -19,8 +19,8 @@ def check_start_state(project, materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(materialized_view) assert isinstance(results, BigQueryMaterializedViewConfig) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 60 + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 60 assert results.partition.field == "record_valid_date" assert results.partition.data_type == "datetime" assert results.partition.granularity == "day" @@ -37,8 +37,8 @@ def check_state_alter_change_is_applied(project, materialized_view): results = project.adapter.describe_relation(materialized_view) assert isinstance(results, BigQueryMaterializedViewConfig) # these change when run manually - assert results.options.enable_refresh is False - assert results.options.refresh_interval_minutes == 30 # BQ returns it to the default + assert results.enable_refresh is False + assert results.refresh_interval_minutes == 30 # BQ returns it to the default @staticmethod def change_config_via_replace(project, materialized_view): diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view_refresh_changes.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view_refresh_changes.py index bef5931a8..dbdd792ed 100644 --- a/tests/functional/adapter/materialized_view_tests/test_materialized_view_refresh_changes.py +++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view_refresh_changes.py @@ -79,10 +79,10 @@ def models(self): def check_start_state(project, my_materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(my_materialized_view) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 30 - assert results.options.max_staleness is None - assert results.options.allow_non_incremental_definition is None + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 30 + assert results.max_staleness is None + assert results.allow_non_incremental_definition is None @staticmethod def change_config(project, my_materialized_view): @@ -97,10 +97,10 @@ def change_config(project, my_materialized_view): def check_end_state(project, my_materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(my_materialized_view) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 30 - assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" - assert results.options.allow_non_incremental_definition is True + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 30 + assert results.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.allow_non_incremental_definition is True class TestTurnOffAllowNonIncremental(RefreshChanges): @@ -115,10 +115,10 @@ def models(self): def check_start_state(project, my_materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(my_materialized_view) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 30 - assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" - assert results.options.allow_non_incremental_definition is True + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 30 + assert results.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.allow_non_incremental_definition is True @staticmethod def change_config(project, my_materialized_view): @@ -132,10 +132,10 @@ def change_config(project, my_materialized_view): def check_end_state(project, my_materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(my_materialized_view) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 30 - assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" - assert results.options.allow_non_incremental_definition is False + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 30 + assert results.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.allow_non_incremental_definition is False class TestUnsetAllowNonIncremental(RefreshChanges): @@ -150,10 +150,10 @@ def models(self): def check_start_state(project, my_materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(my_materialized_view) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 30 - assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" - assert results.options.allow_non_incremental_definition is True + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 30 + assert results.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.allow_non_incremental_definition is True @staticmethod def change_config(project, my_materialized_view): @@ -165,10 +165,10 @@ def change_config(project, my_materialized_view): def check_end_state(project, my_materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(my_materialized_view) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 30 - assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" - assert results.options.allow_non_incremental_definition is None + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 30 + assert results.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.allow_non_incremental_definition is None class TestTurnOffRefresh(RefreshChanges): @@ -183,10 +183,10 @@ def models(self): def check_start_state(project, my_materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(my_materialized_view) - assert results.options.enable_refresh is True - assert results.options.refresh_interval_minutes == 60 - assert results.options.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" - assert results.options.allow_non_incremental_definition is True + assert results.enable_refresh is True + assert results.refresh_interval_minutes == 60 + assert results.max_staleness == "INTERVAL '0-0 0 0:45:0' YEAR TO SECOND" + assert results.allow_non_incremental_definition is True @staticmethod def change_config(project, my_materialized_view): @@ -198,9 +198,9 @@ def change_config(project, my_materialized_view): def check_end_state(project, my_materialized_view): with get_connection(project.adapter): results = project.adapter.describe_relation(my_materialized_view) - assert results.options.enable_refresh is False + assert results.enable_refresh is False assert ( - results.options.refresh_interval_minutes == 30 + results.refresh_interval_minutes == 30 ) # this is a defaulted value in BQ, so it doesn't get cleared - assert results.options.max_staleness is None - assert results.options.allow_non_incremental_definition is None + assert results.max_staleness is None + assert results.allow_non_incremental_definition is None