From 076c5c1ede15044dd2b61f55484b6174eb7de985 Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Tue, 14 Dec 2021 13:38:37 +0900 Subject: [PATCH] Implement dbt models for dbt 1.0 (#32) --- ci/setup.sh | 2 +- dbt_artifacts_models/.gitignore | 1 + dbt_artifacts_models/Makefile | 7 ++- dbt_artifacts_models/dbt_project.yml | 8 +-- .../generic_test_run_results_v4.sql | 47 +++++++++++++++ .../generic_test_run_results_v4/schema.yml | 29 +++++++++ .../last_generic_test_run_results_v4.sql | 30 ++++++++++ .../schema.yml | 30 ++++++++++ .../last_model_run_results_v4.sql | 30 ++++++++++ .../last_model_run_results_v4/schema.yml | 30 ++++++++++ .../last_singular_test_run_results_v4.sql | 30 ++++++++++ .../schema.yml | 30 ++++++++++ .../last_snapshot_run_results_v4.sql | 30 ++++++++++ .../last_snapshot_run_results_v4/schema.yml | 30 ++++++++++ .../model_run_results_v4.sql | 47 +++++++++++++++ .../model_run_results_v4/schema.yml | 29 +++++++++ .../singular_test_run_results_v4/schema.yml | 29 +++++++++ .../singular_test_run_results_v4.sql | 47 +++++++++++++++ .../snapshot_run_results_v4/schema.yml | 29 +++++++++ .../snapshot_run_results_v4.sql | 47 +++++++++++++++ .../models/sources/v3/sources_v3.yml | 14 +++++ .../models/sources/v4/manifest_v4.yml | 14 +++++ .../models/sources/v4/run_results_v4.yml | 14 +++++ .../source_freshness_output_v3/schema.yml | 24 ++++++++ .../source_freshness_output_v3.sql | 39 ++++++++++++ .../schema.yml | 24 ++++++++ .../source_freshness_runtime_error_v3.sql | 39 ++++++++++++ .../parsed_generic_test_node_v4.sql | 57 ++++++++++++++++++ .../parsed_generic_test_node_v4/schema.yml | 24 ++++++++ .../parsed_model_node_v4.sql | 40 +++++++++++++ .../parsed_model_node_v4/schema.yml | 24 ++++++++ .../parsed_singular_test_node_v4.sql | 57 ++++++++++++++++++ .../parsed_singular_test_node_v4/schema.yml | 24 ++++++++ .../parsed_snapshot_node_v4.sql | 57 ++++++++++++++++++ .../parsed_snapshot_node_v4/schema.yml | 24 ++++++++ .../source_manifests_v4/schema.yml | 24 ++++++++ .../source_manifests_v4.sql | 40 +++++++++++++ .../expanded_run_results_v4.sql | 60 +++++++++++++++++++ .../expanded_run_results_v4/schema.yml | 25 ++++++++ dbt_artifacts_models/packages.yml | 2 +- dbt_artifacts_models/selectors.yml | 41 +++++++++++++ requirements/requirements.txt | 2 +- 42 files changed, 1222 insertions(+), 9 deletions(-) create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/generic_test_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/last_generic_test_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/last_model_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/last_singular_test_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/last_snapshot_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/model_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/singular_test_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/snapshot_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/sources/v3/sources_v3.yml create mode 100644 dbt_artifacts_models/models/sources/v4/manifest_v4.yml create mode 100644 dbt_artifacts_models/models/sources/v4/run_results_v4.yml create mode 100644 dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_output_v3/schema.yml create mode 100644 dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_output_v3/source_freshness_output_v3.sql create mode 100644 dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_runtime_error_v3/schema.yml create mode 100644 dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_runtime_error_v3/source_freshness_runtime_error_v3.sql create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_generic_test_node_v4/parsed_generic_test_node_v4.sql create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_generic_test_node_v4/schema.yml create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_model_node_v4/parsed_model_node_v4.sql create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_model_node_v4/schema.yml create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_singular_test_node_v4/parsed_singular_test_node_v4.sql create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_singular_test_node_v4/schema.yml create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_snapshot_node_v4/parsed_snapshot_node_v4.sql create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_snapshot_node_v4/schema.yml create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/source_manifests_v4/schema.yml create mode 100644 dbt_artifacts_models/models/staging/v4/manifest_v4/source_manifests_v4/source_manifests_v4.sql create mode 100644 dbt_artifacts_models/models/staging/v4/run_results_v4/expanded_run_results_v4/expanded_run_results_v4.sql create mode 100644 dbt_artifacts_models/models/staging/v4/run_results_v4/expanded_run_results_v4/schema.yml create mode 100644 dbt_artifacts_models/selectors.yml diff --git a/ci/setup.sh b/ci/setup.sh index 63d2eb0..48224e5 100755 --- a/ci/setup.sh +++ b/ci/setup.sh @@ -32,4 +32,4 @@ else fi # Install required modules. -$PIP install --no-cache-dir --force-reinstall -r "${MODULE_DIR}/requirements/requirements-dev.txt" +$PIP install --no-cache-dir --force-reinstall --use-deprecated=legacy-resolver -r "${MODULE_DIR}/requirements/requirements-dev.txt" diff --git a/dbt_artifacts_models/.gitignore b/dbt_artifacts_models/.gitignore index 7635ab4..a7a41fa 100644 --- a/dbt_artifacts_models/.gitignore +++ b/dbt_artifacts_models/.gitignore @@ -1,5 +1,6 @@ target/ dbt_modules/ +dbt_packages/ logs/ .DS_Store diff --git a/dbt_artifacts_models/Makefile b/dbt_artifacts_models/Makefile index d8c4d35..219a97d 100644 --- a/dbt_artifacts_models/Makefile +++ b/dbt_artifacts_models/Makefile @@ -2,14 +2,17 @@ setup: dbt clean dbt deps +build: + dbt build --profiles-dir ./profiles --selector dbt-1.0 + run: - dbt run --profiles-dir ./profiles + dbt run --profiles-dir ./profiles --selector dbt-1.0 compile: dbt compile --profiles-dir ./profiles test: - dbt test --profiles-dir profiles/ + dbt test --profiles-dir profiles/ --selector dbt-1.0 docs-generate: dbt docs generate --profiles-dir profiles/ diff --git a/dbt_artifacts_models/dbt_project.yml b/dbt_artifacts_models/dbt_project.yml index 8ed631a..967bdb9 100644 --- a/dbt_artifacts_models/dbt_project.yml +++ b/dbt_artifacts_models/dbt_project.yml @@ -2,14 +2,14 @@ name: dbt_artifacts_models profile: dbt-artifacts-loader -version: '1.0' +version: '1.2' config-version: 2 -require-dbt-version: ">=0.19.0" +require-dbt-version: ">=1.0.0" -source-paths: ["models"] +model-paths: ["models"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] target-path: "target" diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/generic_test_run_results_v4.sql b/dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/generic_test_run_results_v4.sql new file mode 100644 index 0000000..74536bf --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/generic_test_run_results_v4.sql @@ -0,0 +1,47 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="generic_test_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH run_results AS ( + SELECT + run_results.*, + (SELECT AS STRUCT generic_tests.*) AS generic_test, + FROM {{ ref("expanded_run_results_v4") }} AS run_results + LEFT OUTER JOIN {{ ref("parsed_generic_test_node_v4") }} AS generic_tests + ON run_results.unique_id = generic_tests.unique_id + AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, generic_tests.metadata.generated_at, DAY)) <= 2 + WHERE generic_tests.unique_id IS NOT NULL + AND timing_name IN ("execute") +) +-- Extract only run results whose metadata is the most close to that of model. +, nearest_manifests AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY unique_id, metadata.invocation_id ORDER BY generated_at_diff) AS rank, + * EXCEPT(generated_at_diff) + FROM ( + SELECT + ABS(DATETIME_DIFF(metadata.generated_at, generic_test.metadata.generated_at, SECOND)) AS generated_at_diff, + *, + FROM run_results + ) +) + +SELECT + * EXCEPT(rank) +FROM nearest_manifests +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/schema.yml b/dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/schema.yml new file mode 100644 index 0000000..21cec1c --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/schema.yml @@ -0,0 +1,29 @@ +--- +version: 2 + +models: + - name: generic_test_run_results_v4 + description: | + This is a denormalized schema test run results v4. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null + - name: timing_name + description: "timing name" + tests: + - accepted_values: + values: ["execute"] diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/last_generic_test_run_results_v4.sql b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/last_generic_test_run_results_v4.sql new file mode 100644 index 0000000..c8c19b1 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/last_generic_test_run_results_v4.sql @@ -0,0 +1,30 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="last_generic_test_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH run_results AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY unique_id ORDER BY completed_at DESC) AS rank, + *, + FROM {{ ref("generic_test_run_results_v4") }} +) + +SELECT + * EXCEPT(rank) +FROM run_results +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/schema.yml b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/schema.yml new file mode 100644 index 0000000..3a78dc5 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/schema.yml @@ -0,0 +1,30 @@ +--- +version: 2 + +models: + - name: last_generic_test_run_results_v4 + description: | + The table contains only the latest record of data test run results v4. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null + - unique + - name: timing_name + description: "timing name" + tests: + - accepted_values: + values: ["execute"] diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/last_model_run_results_v4.sql b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/last_model_run_results_v4.sql new file mode 100644 index 0000000..4d60463 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/last_model_run_results_v4.sql @@ -0,0 +1,30 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="last_model_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH run_results AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY unique_id ORDER BY completed_at DESC) AS rank, + *, + FROM {{ ref("model_run_results_v4") }} +) + +SELECT + * EXCEPT(rank) +FROM run_results +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/schema.yml b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/schema.yml new file mode 100644 index 0000000..5b3655b --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/schema.yml @@ -0,0 +1,30 @@ +--- +version: 2 + +models: + - name: last_model_run_results_v4 + description: | + The table contains only the latest record of model run results v4. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null + - unique + - name: timing_name + description: "timing name" + tests: + - accepted_values: + values: ["execute"] diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/last_singular_test_run_results_v4.sql b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/last_singular_test_run_results_v4.sql new file mode 100644 index 0000000..296ee76 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/last_singular_test_run_results_v4.sql @@ -0,0 +1,30 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="last_singular_test_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH run_results AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY unique_id ORDER BY completed_at DESC) AS rank, + *, + FROM {{ ref("singular_test_run_results_v4") }} +) + +SELECT + * EXCEPT(rank) +FROM run_results +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/schema.yml b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/schema.yml new file mode 100644 index 0000000..77c78d1 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/schema.yml @@ -0,0 +1,30 @@ +--- +version: 2 + +models: + - name: last_singular_test_run_results_v4 + description: | + The table contains only the latest record of schema test run results v4. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null + - unique + - name: timing_name + description: "timing name" + tests: + - accepted_values: + values: ["execute"] diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/last_snapshot_run_results_v4.sql b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/last_snapshot_run_results_v4.sql new file mode 100644 index 0000000..a600cc9 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/last_snapshot_run_results_v4.sql @@ -0,0 +1,30 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="last_snapshot_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH run_results AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY unique_id ORDER BY completed_at DESC) AS rank, + *, + FROM {{ ref("snapshot_run_results_v4") }} +) + +SELECT + * EXCEPT(rank) +FROM run_results +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/schema.yml b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/schema.yml new file mode 100644 index 0000000..c7be880 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/schema.yml @@ -0,0 +1,30 @@ +--- +version: 2 + +models: + - name: last_snapshot_run_results_v4 + description: | + The table contains only the latest record of snapshot run results v4. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null + - unique + - name: timing_name + description: "timing name" + tests: + - accepted_values: + values: ["execute"] diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/model_run_results_v4.sql b/dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/model_run_results_v4.sql new file mode 100644 index 0000000..0113fb6 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/model_run_results_v4.sql @@ -0,0 +1,47 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="model_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH run_results AS ( + SELECT + run_results.*, + (SELECT AS STRUCT models.*) AS model, + FROM {{ ref("expanded_run_results_v4") }} AS run_results + LEFT OUTER JOIN {{ ref("parsed_model_node_v4") }} AS models + ON run_results.unique_id = models.unique_id + AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, models.metadata.generated_at, DAY)) <= 2 + WHERE models.unique_id IS NOT NULL + AND timing_name IN ("execute") +) +-- Extract only run results whose metadata is the most close to that of model. +, nearest_manifests AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY unique_id, metadata.invocation_id ORDER BY generated_at_diff) AS rank, + * EXCEPT(generated_at_diff) + FROM ( + SELECT + ABS(DATETIME_DIFF(metadata.generated_at, model.metadata.generated_at, SECOND)) AS generated_at_diff, + *, + FROM run_results + ) +) + +SELECT + * EXCEPT(rank) +FROM nearest_manifests +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/schema.yml b/dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/schema.yml new file mode 100644 index 0000000..3196f09 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/schema.yml @@ -0,0 +1,29 @@ +--- +version: 2 + +models: + - name: model_run_results_v4 + description: | + This is a denormalized model run results v4. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null + - name: timing_name + description: "timing name" + tests: + - accepted_values: + values: ["execute"] diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/schema.yml b/dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/schema.yml new file mode 100644 index 0000000..e6502d1 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/schema.yml @@ -0,0 +1,29 @@ +--- +version: 2 + +models: + - name: singular_test_run_results_v4 + description: | + This is a denormalized data test run results v4. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null + - name: timing_name + description: "timing name" + tests: + - accepted_values: + values: ["execute"] diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/singular_test_run_results_v4.sql b/dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/singular_test_run_results_v4.sql new file mode 100644 index 0000000..c797c4d --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/singular_test_run_results_v4.sql @@ -0,0 +1,47 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="singular_test_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH run_results AS ( + SELECT + run_results.*, + (SELECT AS STRUCT singular_tests.*) AS singular_test, + FROM {{ ref("expanded_run_results_v4") }} AS run_results + LEFT OUTER JOIN {{ ref("parsed_singular_test_node_v4") }} AS singular_tests + ON run_results.unique_id = singular_tests.unique_id + AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, singular_tests.metadata.generated_at, DAY)) <= 2 + WHERE singular_tests.unique_id IS NOT NULL + AND timing_name IN ("execute") +) +-- Extract only run results whose metadata is the most close to that of model. +, nearest_manifests AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY unique_id, metadata.invocation_id ORDER BY generated_at_diff) AS rank, + * EXCEPT(generated_at_diff) + FROM ( + SELECT + ABS(DATETIME_DIFF(metadata.generated_at, singular_test.metadata.generated_at, SECOND)) AS generated_at_diff, + *, + FROM run_results + ) +) + +SELECT + * EXCEPT(rank) +FROM nearest_manifests +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/schema.yml b/dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/schema.yml new file mode 100644 index 0000000..73c18f8 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/schema.yml @@ -0,0 +1,29 @@ +--- +version: 2 + +models: + - name: snapshot_run_results_v4 + description: | + This is a denormalized snapshot run results v4. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null + - name: timing_name + description: "timing name" + tests: + - accepted_values: + values: ["execute"] diff --git a/dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/snapshot_run_results_v4.sql b/dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/snapshot_run_results_v4.sql new file mode 100644 index 0000000..bdb9894 --- /dev/null +++ b/dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/snapshot_run_results_v4.sql @@ -0,0 +1,47 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="snapshot_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH run_results AS ( + SELECT + run_results.*, + (SELECT AS STRUCT snapshots.*) AS snapshot, + FROM {{ ref("expanded_run_results_v4") }} AS run_results + LEFT OUTER JOIN {{ ref("parsed_snapshot_node_v4") }} AS snapshots + ON run_results.unique_id = snapshots.unique_id + AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, snapshots.metadata.generated_at, DAY)) <= 2 + WHERE snapshots.unique_id IS NOT NULL + AND timing_name IN ("execute") +) +-- Extract only run results whose metadata is the most close to that of model. +, nearest_manifests AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY unique_id, metadata.invocation_id ORDER BY generated_at_diff) AS rank, + * EXCEPT(generated_at_diff) + FROM ( + SELECT + ABS(DATETIME_DIFF(metadata.generated_at, snapshot.metadata.generated_at, SECOND)) AS generated_at_diff, + *, + FROM run_results + ) +) + +SELECT + * EXCEPT(rank) +FROM nearest_manifests +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/sources/v3/sources_v3.yml b/dbt_artifacts_models/models/sources/v3/sources_v3.yml new file mode 100644 index 0000000..eafa0e8 --- /dev/null +++ b/dbt_artifacts_models/models/sources/v3/sources_v3.yml @@ -0,0 +1,14 @@ +--- +version: 2 + +sources: + - name: "{{ var('dbt_artifacts_loader')['dataset'] }}" + database: "{{ var('dbt_artifacts_loader')['project'] }}" + tables: + - name: sources_v3 + identifier: sources_v3 + + columns: + - name: loaded_at + tests: + - not_null diff --git a/dbt_artifacts_models/models/sources/v4/manifest_v4.yml b/dbt_artifacts_models/models/sources/v4/manifest_v4.yml new file mode 100644 index 0000000..5f98156 --- /dev/null +++ b/dbt_artifacts_models/models/sources/v4/manifest_v4.yml @@ -0,0 +1,14 @@ +--- +version: 2 + +sources: + - name: "{{ var('dbt_artifacts_loader')['dataset'] }}" + database: "{{ var('dbt_artifacts_loader')['project'] }}" + tables: + - name: manifest_v4 + identifier: manifest_v4 + + columns: + - name: loaded_at + tests: + - not_null diff --git a/dbt_artifacts_models/models/sources/v4/run_results_v4.yml b/dbt_artifacts_models/models/sources/v4/run_results_v4.yml new file mode 100644 index 0000000..986d969 --- /dev/null +++ b/dbt_artifacts_models/models/sources/v4/run_results_v4.yml @@ -0,0 +1,14 @@ +--- +version: 2 + +sources: + - name: "{{ var('dbt_artifacts_loader')['dataset'] }}" + database: "{{ var('dbt_artifacts_loader')['project'] }}" + tables: + - name: run_results_v4 + identifier: run_results_v4 + + columns: + - name: loaded_at + tests: + - not_null diff --git a/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_output_v3/schema.yml b/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_output_v3/schema.yml new file mode 100644 index 0000000..71ac156 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_output_v3/schema.yml @@ -0,0 +1,24 @@ +--- +version: 2 + +models: + - name: source_freshness_output_v3 + description: | + This is an expanded `sources_v3`. + If there are duplicated results, the latest results are taken. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - invocation_id + - unique_id + + columns: + - name: invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null diff --git a/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_output_v3/source_freshness_output_v3.sql b/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_output_v3/source_freshness_output_v3.sql new file mode 100644 index 0000000..c66c0e7 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_output_v3/source_freshness_output_v3.sql @@ -0,0 +1,39 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="source_freshness_output_v3", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH expanded_sources AS ( + SELECT + loaded_at AS loaded_at, + elapsed_time, + metadata.*, + result.values.SourceFreshnessOutput.*, + FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'sources_v3') }} + , UNNEST(results) AS result +) +, remove_duplicates AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY invocation_id, unique_id ORDER BY generated_at DESC) AS rank, + *, + FROM expanded_sources +) + +SELECT + * EXCEPT(rank) +FROM remove_duplicates +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_runtime_error_v3/schema.yml b/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_runtime_error_v3/schema.yml new file mode 100644 index 0000000..6c5784a --- /dev/null +++ b/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_runtime_error_v3/schema.yml @@ -0,0 +1,24 @@ +--- +version: 2 + +models: + - name: source_freshness_runtime_error_v3 + description: | + This is an expanded `sources_v3`. + If there are duplicated results, the latest results are taken. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - invocation_id + - unique_id + + columns: + - name: invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null diff --git a/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_runtime_error_v3/source_freshness_runtime_error_v3.sql b/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_runtime_error_v3/source_freshness_runtime_error_v3.sql new file mode 100644 index 0000000..8c471ac --- /dev/null +++ b/dbt_artifacts_models/models/staging/v3/sources_v3/source_freshness_runtime_error_v3/source_freshness_runtime_error_v3.sql @@ -0,0 +1,39 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="source_freshness_runtime_error_v3", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH expanded_sources AS ( + SELECT + loaded_at AS loaded_at, + elapsed_time, + metadata.*, + result.values.SourceFreshnessOutput.*, + FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'sources_v3') }} + , UNNEST(results) AS result +) +, remove_duplicates AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY invocation_id, unique_id ORDER BY generated_at DESC) AS rank, + *, + FROM expanded_sources +) + +SELECT + * EXCEPT(rank) +FROM remove_duplicates +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_generic_test_node_v4/parsed_generic_test_node_v4.sql b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_generic_test_node_v4/parsed_generic_test_node_v4.sql new file mode 100644 index 0000000..0af9958 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_generic_test_node_v4/parsed_generic_test_node_v4.sql @@ -0,0 +1,57 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="parsed_generic_test_node_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH generic_tests AS ( + SELECT + t.* EXCEPT(depends_on), + depends_on_node, + depends_on_macro, + FROM ( + SELECT + loaded_at AS loaded_at, + metadata AS metadata, + node.key AS key, + node.value.ParsedGenericTestNode.*, + FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'manifest_v4') }} + , UNNEST(nodes) AS node + ) AS t + CROSS JOIN UNNEST(depends_on.nodes.value) AS depends_on_node + CROSS JOIN UNNEST(depends_on.macros.value) AS depends_on_macro +) +, generic_tests_with_models AS ( + SELECT + generic_tests.*, + (SELECT AS STRUCT models.*) AS depends_on_model, + FROM generic_tests AS generic_tests + LEFT OUTER JOIN {{ ref("parsed_model_node_v4") }} AS models + ON generic_tests.metadata.invocation_id = models.metadata.invocation_id + AND generic_tests.depends_on_node = models.unique_id +) +, remove_duplicates AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY metadata.invocation_id, unique_id ORDER BY metadata.generated_at DESC) AS rank, + * + FROM generic_tests_with_models + WHERE unique_id IS NOT NULL +) + +SELECT + * EXCEPT(rank) +FROM remove_duplicates +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_generic_test_node_v4/schema.yml b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_generic_test_node_v4/schema.yml new file mode 100644 index 0000000..6bcbdbd --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_generic_test_node_v4/schema.yml @@ -0,0 +1,24 @@ +--- +version: 2 + +models: + - name: parsed_generic_test_node_v4 + description: | + This is a table to expand `manifest_v4.node.value.ParsedGenericTestNode`. + The table exclude duplicates. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique ID" + tests: + - not_null diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_model_node_v4/parsed_model_node_v4.sql b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_model_node_v4/parsed_model_node_v4.sql new file mode 100644 index 0000000..a2c68de --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_model_node_v4/parsed_model_node_v4.sql @@ -0,0 +1,40 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="parsed_model_node_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH expanded_artifacts AS ( + SELECT + loaded_at AS loaded_at, + metadata AS metadata, + node.key AS key, + node.value.ParsedModelNode.*, + FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'manifest_v4') }} + , UNNEST(nodes) AS node +) +, remove_duplicates AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY metadata.invocation_id, unique_id ORDER BY metadata.generated_at DESC) AS rank, + * + FROM expanded_artifacts + WHERE unique_id IS NOT NULL +) + +SELECT + * EXCEPT(rank) +FROM remove_duplicates +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_model_node_v4/schema.yml b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_model_node_v4/schema.yml new file mode 100644 index 0000000..6136e30 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_model_node_v4/schema.yml @@ -0,0 +1,24 @@ +--- +version: 2 + +models: + - name: parsed_model_node_v4 + description: | + This is a table to expand `manifest_v4.node.value.ParsedModelNode`. + The table exclude duplicates. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique ID" + tests: + - not_null diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_singular_test_node_v4/parsed_singular_test_node_v4.sql b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_singular_test_node_v4/parsed_singular_test_node_v4.sql new file mode 100644 index 0000000..76addcb --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_singular_test_node_v4/parsed_singular_test_node_v4.sql @@ -0,0 +1,57 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="parsed_singular_test_node_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH singular_tests AS ( + SELECT + s.* EXCEPT(depends_on), + depends_on_node, + depends_on_macro, + FROM ( + SELECT + loaded_at AS loaded_at, + metadata AS metadata, + node.key AS key, + node.value.ParsedSingularTestNode.*, + FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'manifest_v4') }} + , UNNEST(nodes) AS node + ) AS s + CROSS JOIN UNNEST(depends_on.nodes.value) AS depends_on_node + CROSS JOIN UNNEST(depends_on.macros.value) AS depends_on_macro +) +, singular_tests_with_models AS ( + SELECT + singular_tests.*, + (SELECT AS STRUCT models.*) AS depends_on_model, + FROM singular_tests AS singular_tests + LEFT OUTER JOIN {{ ref("parsed_model_node_v4") }} AS models + ON singular_tests.metadata.invocation_id = models.metadata.invocation_id + AND singular_tests.depends_on_node = models.unique_id +) +, remove_duplicates AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY metadata.invocation_id, unique_id ORDER BY metadata.generated_at DESC) AS rank, + * + FROM singular_tests_with_models + WHERE unique_id IS NOT NULL +) + +SELECT + * EXCEPT(rank) +FROM remove_duplicates +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_singular_test_node_v4/schema.yml b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_singular_test_node_v4/schema.yml new file mode 100644 index 0000000..ad577b8 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_singular_test_node_v4/schema.yml @@ -0,0 +1,24 @@ +--- +version: 2 + +models: + - name: parsed_singular_test_node_v4 + description: | + This is a table to expand `manifest_v4.node.value.ParsedSingularTestNode`. + The table exclude duplicates. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique ID" + tests: + - not_null diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_snapshot_node_v4/parsed_snapshot_node_v4.sql b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_snapshot_node_v4/parsed_snapshot_node_v4.sql new file mode 100644 index 0000000..9385816 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_snapshot_node_v4/parsed_snapshot_node_v4.sql @@ -0,0 +1,57 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="parsed_snapshot_node_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH snapshots AS ( + SELECT + s.* EXCEPT(depends_on), + depends_on_node, + depends_on_macro, + FROM ( + SELECT + loaded_at AS loaded_at, + metadata AS metadata, + node.key AS key, + node.value.ParsedSnapshotNode.*, + FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'manifest_v4') }} + , UNNEST(nodes) AS node + ) AS s + CROSS JOIN UNNEST(depends_on.nodes.value) AS depends_on_node + CROSS JOIN UNNEST(depends_on.macros.value) AS depends_on_macro +) +, snapshots_with_models AS ( + SELECT + snapshots.*, + (SELECT AS STRUCT models.*) AS depends_on_model, + FROM snapshots AS snapshots + LEFT OUTER JOIN {{ ref("parsed_model_node_v4") }} AS models + ON snapshots.metadata.invocation_id = models.metadata.invocation_id + AND snapshots.depends_on_node = models.unique_id +) +, remove_duplicates AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY metadata.invocation_id, unique_id ORDER BY metadata.generated_at DESC) AS rank, + * + FROM snapshots_with_models + WHERE unique_id IS NOT NULL +) + +SELECT + * EXCEPT(rank) +FROM remove_duplicates +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_snapshot_node_v4/schema.yml b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_snapshot_node_v4/schema.yml new file mode 100644 index 0000000..14a06d3 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/parsed_snapshot_node_v4/schema.yml @@ -0,0 +1,24 @@ +--- +version: 2 + +models: + - name: parsed_snapshot_node_v4 + description: | + This is a table to expand `manifest_v4.node.value.ParsedSnapshotNode`. + The table exclude duplicates. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique ID" + tests: + - not_null diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/source_manifests_v4/schema.yml b/dbt_artifacts_models/models/staging/v4/manifest_v4/source_manifests_v4/schema.yml new file mode 100644 index 0000000..e893b29 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/source_manifests_v4/schema.yml @@ -0,0 +1,24 @@ +--- +version: 2 + +models: + - name: source_manifests_v4 + description: | + This is a table to expand `manifest_v4.sources.value`. + The table exclude duplicates. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique ID" + tests: + - not_null diff --git a/dbt_artifacts_models/models/staging/v4/manifest_v4/source_manifests_v4/source_manifests_v4.sql b/dbt_artifacts_models/models/staging/v4/manifest_v4/source_manifests_v4/source_manifests_v4.sql new file mode 100644 index 0000000..bfff90d --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/manifest_v4/source_manifests_v4/source_manifests_v4.sql @@ -0,0 +1,40 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="source_manifests_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH expanded_artifacts AS ( + SELECT + loaded_at AS loaded_at, + metadata AS metadata, + source.key AS key, + source.value.* + FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'manifest_v4') }} + , UNNEST(sources) AS source +) +, remove_duplicates AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY metadata.invocation_id, unique_id ORDER BY metadata.generated_at DESC) AS rank, + * + FROM expanded_artifacts + WHERE unique_id IS NOT NULL +) + +SELECT + * EXCEPT(rank) +FROM remove_duplicates +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/staging/v4/run_results_v4/expanded_run_results_v4/expanded_run_results_v4.sql b/dbt_artifacts_models/models/staging/v4/run_results_v4/expanded_run_results_v4/expanded_run_results_v4.sql new file mode 100644 index 0000000..fa9fe68 --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/run_results_v4/expanded_run_results_v4/expanded_run_results_v4.sql @@ -0,0 +1,60 @@ +{% set project = var('dbt_artifacts_loader')['project'] %} +{% set dataset = var('dbt_artifacts_loader')['dataset'] %} + +{{ + config( + enabled=true, + full_refresh=none, + materialized="view", + database=project, + schema=dataset, + alias="expanded_run_results_v4", + persist_docs={"relation": true, "columns": true}, + labels={ + "modeled_by": "dbt", + "app": "dbt-artifacts-loader", + }, + ) +}} + +WITH expanded_results AS ( + SELECT + loaded_at AS loaded_at, + STRUCT( + JSON_EXTRACT_SCALAR(args, "$.selector_name") AS selector_name, + JSON_EXTRACT_SCALAR(args, "$.profile") AS profile, + JSON_EXTRACT_SCALAR(args, "$.target") AS target, + JSON_EXTRACT_SCALAR(args, "$.rpc_method") AS rpc_method, + JSON_EXTRACT_SCALAR(args, "$.vars") AS vars + ) AS args, + args AS raw_args, + metadata, + result.adapter_response AS adapter_response, + result.unique_id AS unique_id, + result.status AS status, + result.execution_time AS execution_time, + result.message AS message, + result.timing AS timing, + FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'run_results_v4') }} + , UNNEST(results) AS result +) +, expanded_timing AS ( + SELECT + fr.* EXCEPT (timing), + expanded_timing.name AS timing_name, + expanded_timing.completed_at AS completed_at, + expanded_timing.started_at AS started_at, + FROM expanded_results AS fr + , UNNEST(timing) AS expanded_timing +) +, remove_duplicates AS ( + SELECT + ROW_NUMBER() OVER (PARTITION BY metadata.invocation_id, unique_id, timing_name ORDER BY metadata.generated_at DESC) AS rank, + * + FROM expanded_timing +) + +SELECT + * EXCEPT(rank) +FROM remove_duplicates +WHERE rank = 1 diff --git a/dbt_artifacts_models/models/staging/v4/run_results_v4/expanded_run_results_v4/schema.yml b/dbt_artifacts_models/models/staging/v4/run_results_v4/expanded_run_results_v4/schema.yml new file mode 100644 index 0000000..464e5af --- /dev/null +++ b/dbt_artifacts_models/models/staging/v4/run_results_v4/expanded_run_results_v4/schema.yml @@ -0,0 +1,25 @@ +--- +version: 2 + +models: + - name: expanded_run_results_v4 + description: | + This is an expanded `run_results_v4`. + If there are duplicated results, the latest results are taken. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - metadata.invocation_id + - unique_id + - timing_name + + columns: + - name: metadata.invocation_id + description: "invocation ID" + tests: + - not_null + - name: unique_id + description: "unique test ID" + tests: + - not_null diff --git a/dbt_artifacts_models/packages.yml b/dbt_artifacts_models/packages.yml index 6e9f115..d9261a8 100644 --- a/dbt_artifacts_models/packages.yml +++ b/dbt_artifacts_models/packages.yml @@ -1,3 +1,3 @@ packages: - package: dbt-labs/dbt_utils - version: 0.7.1 + version: 0.8.0 diff --git a/dbt_artifacts_models/selectors.yml b/dbt_artifacts_models/selectors.yml new file mode 100644 index 0000000..abacd35 --- /dev/null +++ b/dbt_artifacts_models/selectors.yml @@ -0,0 +1,41 @@ +selectors: + + # For dbt==0.2x + - name: dbt-0.2x + definition: + union: + # Sources + - "models/sources/v3/sources_v2.yml" + - "models/sources/v4/manifest_v3.yml" + - "models/sources/v4/run_results_v3.yml" + - "models/sources/v1/catalog_v1.yml" + # staging + - "models/staging/v3/sources_v2/*" + - "models/staging/v4/manifest_v3/*" + - "models/staging/v4/run_results_v3/*" + - "models/staging/v1/catalog_v1/*" + # mart + - "models/marts/v3/sources_v2/*" + - "models/marts/v4/manifest_v3/*" + - "models/marts/v4/run_results_v3/*" + - "models/marts/v1/catalog_v1/*" + + # For dbt==1.0 + - name: dbt-1.0 + definition: + union: + # Sources + - "models/sources/v3/sources_v3.yml" + - "models/sources/v4/manifest_v4.yml" + - "models/sources/v4/run_results_v4.yml" + - "models/sources/v1/catalog_v1.yml" + # staging + - "models/staging/v3/sources_v3/*" + - "models/staging/v4/manifest_v4/*" + - "models/staging/v4/run_results_v4/*" + - "models/staging/v1/catalog_v1/*" + # mart + - "models/marts/v3/sources_v3/*" + - "models/marts/v4/manifest_v4/*" + - "models/marts/v4/run_results_v4/*" + - "models/marts/v1/catalog_v1/*" diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 8d7c89c..e36b47c 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -11,7 +11,7 @@ python-dotenv==0.17.0 python-json-logger==2.0.1 # dbt -dbt-bigquery==0.20.1 +dbt-bigquery==1.0.0 # Utils datamodel-code-generator==0.11.13