From d5b4114ca1ef28d3c2bd3c61812e3a2c901a2366 Mon Sep 17 00:00:00 2001 From: kodaho <23268819+kodaho@users.noreply.github.com> Date: Fri, 6 Oct 2023 20:26:53 +0200 Subject: [PATCH 1/4] [fix] Use rendered query comment for job labels (#955) * [fix] Use rendered query comment for job labels * Add test from dbt-labs/dbt-query#872 * Valid JSON in test * Add changie entry --- .../unreleased/Fixes-20231005-235950.yaml | 6 +++ dbt/adapters/bigquery/connections.py | 5 +- .../query_comment_test/test_job_label.py | 52 +++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20231005-235950.yaml create mode 100644 tests/functional/adapter/query_comment_test/test_job_label.py diff --git a/.changes/unreleased/Fixes-20231005-235950.yaml b/.changes/unreleased/Fixes-20231005-235950.yaml new file mode 100644 index 000000000..bf0bf6fa6 --- /dev/null +++ b/.changes/unreleased/Fixes-20231005-235950.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix issue where job labels are not rendered when using macro for query comment +time: 2023-10-05T23:59:50.077842+02:00 +custom: + Author: kodaho mikealfare + Issue: "863" diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index a5c7b9355..1e96ed5ef 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -451,9 +451,10 @@ def raw_execute( hasattr(self.profile, "query_comment") and self.profile.query_comment and self.profile.query_comment.job_label + and self.query_header + and (query_comment := self.query_header.comment.query_comment) ): - query_comment = self.profile.query_comment - labels = self._labels_from_query_comment(query_comment.comment) + labels = self._labels_from_query_comment(query_comment) else: labels = {} diff --git a/tests/functional/adapter/query_comment_test/test_job_label.py b/tests/functional/adapter/query_comment_test/test_job_label.py new file mode 100644 index 000000000..af984a8c4 --- /dev/null +++ b/tests/functional/adapter/query_comment_test/test_job_label.py @@ -0,0 +1,52 @@ +import pytest + +from google.cloud.bigquery.client import Client + +from dbt.tests.util import run_dbt + + +_MACRO__BQ_LABELS = """ +{% macro bq_labels() %}{ + "system": "{{ env_var('LABEL_SYSTEM', 'my_system') }}", + "env_type": "{{ env_var('LABEL_ENV', 'dev') }}" +}{% endmacro %} +""" +_MODEL__MY_TABLE = """ +{{ config(materialized= "table") }} +select 1 as id +""" + + +class TestQueryCommentJobLabel: + @pytest.fixture(scope="class") + def models(self): + return {"my_table.sql": _MODEL__MY_TABLE} + + @pytest.fixture(scope="class") + def macros(self): + return {"bq_labels.sql": _MACRO__BQ_LABELS} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "query-comment": { + "comment": "{{ bq_labels() }}", + "job-label": True, + "append": True, + } + } + + def test_query_comments_displays_as_job_labels(self, project): + """ + Addresses this regression in dbt-bigquery 1.6: + https://github.com/dbt-labs/dbt-bigquery/issues/863 + """ + results = run_dbt(["run"]) + job_id = results.results[0].adapter_response.get("job_id") + with project.adapter.connection_named("_test"): + client: Client = project.adapter.connections.get_thread_connection().handle + job = client.get_job(job_id=job_id) + + # this is what should happen + assert job.labels.get("system") == "my_system" + assert job.labels.get("env_type") == "dev" From 3e8d389e65a67bad0ae467761c4a4519507e367d Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Fri, 6 Oct 2023 16:08:24 -0500 Subject: [PATCH 2/4] a slight rework of 955 to keep a seperation of concerns (#957) * a slight rework of 955 to keep a seperation of concerns * move query_header back into inital if, save assignement for truthy, else return empty dict --- dbt/adapters/bigquery/connections.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index 1e96ed5ef..ff544f0d0 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -435,6 +435,18 @@ def get_table_from_response(cls, resp): column_names = [field.name for field in resp.schema] return agate_helper.table_from_data_flat(resp, column_names) + def get_labels_from_query_comment(cls): + if ( + hasattr(cls.profile, "query_comment") + and cls.profile.query_comment + and cls.profile.query_comment.job_label + and cls.query_header + ): + query_comment = cls.query_header.comment.query_comment + return cls._labels_from_query_comment(query_comment) + + return {} + def raw_execute( self, sql, @@ -447,16 +459,7 @@ def raw_execute( fire_event(SQLQuery(conn_name=conn.name, sql=sql, node_info=get_node_info())) - if ( - hasattr(self.profile, "query_comment") - and self.profile.query_comment - and self.profile.query_comment.job_label - and self.query_header - and (query_comment := self.query_header.comment.query_comment) - ): - labels = self._labels_from_query_comment(query_comment) - else: - labels = {} + labels = self.get_labels_from_query_comment() if active_user: labels["dbt_invocation_id"] = active_user.invocation_id From 4976d54e276340b2df348276d2d0141c041783ce Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 9 Oct 2023 22:05:12 -0700 Subject: [PATCH 3/4] test against 1.7.latest release branch and fix semver (#961) * use dynamic schema in test_grant_access_to.py * use dynamic schema in test_grant_access_to.py * revert setup * test against 1.6.latest release branch and fix semver * test against 1.6.latest --- .github/workflows/nightly-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 46db5b749..ef210dacd 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -26,7 +26,7 @@ defaults: shell: bash env: - RELEASE_BRANCH: "1.5.latest" # must test against most recent .latest branch to have parity for dependency with core + RELEASE_BRANCH: "1.6.latest" # must test against most recent .latest branch to have parity for dependency with core jobs: aggregate-release-data: @@ -75,7 +75,7 @@ jobs: - name: "Generate Nightly Release Version Number" id: nightly-release-version run: | - number="${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.${{ steps.bump_patch.outputs.patch }}.dev${{ steps.current-date.outputs.date }}" + number="${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.${{ steps.bump_patch.outputs.patch }}+dev${{ steps.current-date.outputs.date }}" echo "number=$number" >> $GITHUB_OUTPUT - name: "Audit Nightly Release Version And Parse Into Parts" From 5396aa0918abe033a2c677579052928ab164c897 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 10 Oct 2023 13:24:38 -0400 Subject: [PATCH 4/4] Support type aliasing in contracts (#954) --- .../Under the Hood-20231005-115800.yaml | 6 ++++++ dbt/adapters/bigquery/column.py | 5 +---- .../adapter/constraints/test_constraints.py | 17 +++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20231005-115800.yaml diff --git a/.changes/unreleased/Under the Hood-20231005-115800.yaml b/.changes/unreleased/Under the Hood-20231005-115800.yaml new file mode 100644 index 000000000..a5b56ed72 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20231005-115800.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Support for use of type aliases in contract column data_type +time: 2023-10-05T11:58:00.719136-04:00 +custom: + Author: gshank + Issue: "953" diff --git a/dbt/adapters/bigquery/column.py b/dbt/adapters/bigquery/column.py index 1820a6ba7..1bdf4323d 100644 --- a/dbt/adapters/bigquery/column.py +++ b/dbt/adapters/bigquery/column.py @@ -13,12 +13,9 @@ @dataclass(init=False) class BigQueryColumn(Column): TYPE_LABELS = { - "STRING": "STRING", - "TIMESTAMP": "TIMESTAMP", + "TEXT": "STRING", "FLOAT": "FLOAT64", "INTEGER": "INT64", - "BOOLEAN": "BOOLEAN", - "RECORD": "RECORD", } fields: List[Self] # type: ignore mode: str # type: ignore diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py index c8c0c11e2..013f2948b 100644 --- a/tests/functional/adapter/constraints/test_constraints.py +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -47,7 +47,7 @@ _expected_sql_bigquery = """ create or replace table ( - id integer not null primary key not enforced references (id) not enforced, + id INT64 not null primary key not enforced references (id) not enforced, color string, date_day string ) @@ -79,12 +79,9 @@ """ # Different on BigQuery: -# - does not support a data type named 'text' (TODO handle this via type translation/aliasing!) -constraints_yml = model_schema_yml.replace("text", "string") -model_constraints_yml = constrained_model_schema_yml.replace("text", "string") -model_contract_header_schema_yml = model_contract_header_schema_yml.replace("text", "string") -model_fk_constraint_schema_yml = model_fk_constraint_schema_yml.replace("text", "string") -constrained_model_schema_yml = constrained_model_schema_yml.replace("text", "string") +# Switch from text to string handled by aliasing +constraints_yml = model_schema_yml +model_constraints_yml = constrained_model_schema_yml my_model_contract_sql_header_sql = """ {{ @@ -334,7 +331,7 @@ def models(self): def expected_sql(self): return """ create or replace table ( - id integer not null, + id INT64 not null, color string, date_day string, primary key (id) not enforced, @@ -361,14 +358,14 @@ class TestBigQueryConstraintQuotedColumn(BaseConstraintQuotedColumn): def models(self): return { "my_model.sql": my_model_with_quoted_column_name_sql, - "constraints_schema.yml": model_quoted_column_schema_yml.replace("text", "string"), + "constraints_schema.yml": model_quoted_column_schema_yml, } @pytest.fixture(scope="class") def expected_sql(self): return """ create or replace table ( - id integer not null, + id INT64 not null, `from` string not null, date_day string )