Implement dbt models for dbt 1.0 (#32)

ci/setup.sh

@@ -32,4 +32,4 @@ else
 fi
 
 # Install required modules.
-$PIP install --no-cache-dir --force-reinstall -r "${MODULE_DIR}/requirements/requirements-dev.txt"
+$PIP install --no-cache-dir --force-reinstall --use-deprecated=legacy-resolver -r "${MODULE_DIR}/requirements/requirements-dev.txt"

dbt_artifacts_models/.gitignore

@@ -1,5 +1,6 @@
 
 target/
 dbt_modules/
+dbt_packages/
 logs/
 .DS_Store

dbt_artifacts_models/Makefile

@@ -2,14 +2,17 @@ setup:
 	dbt clean
 	dbt deps
 
+build:
+	dbt build --profiles-dir ./profiles --selector dbt-1.0
+
 run:
-	dbt run --profiles-dir ./profiles
+	dbt run --profiles-dir ./profiles --selector dbt-1.0
 
 compile:
 	dbt compile --profiles-dir ./profiles
 
 test:
-	dbt test --profiles-dir profiles/
+	dbt test --profiles-dir profiles/ --selector dbt-1.0
 
 docs-generate:
 	dbt docs generate --profiles-dir profiles/

dbt_artifacts_models/dbt_project.yml

@@ -2,14 +2,14 @@ name: dbt_artifacts_models
 
 profile: dbt-artifacts-loader
 
-version: '1.0'
+version: '1.2'
 config-version: 2
-require-dbt-version: ">=0.19.0"
+require-dbt-version: ">=1.0.0"
 
-source-paths: ["models"]
+model-paths: ["models"]
 analysis-paths: ["analysis"]
 test-paths: ["tests"]
-data-paths: ["data"]
+seed-paths: ["data"]
 macro-paths: ["macros"]
 
 target-path: "target"

dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/generic_test_run_results_v4.sql

@@ -0,0 +1,47 @@
+{% set project = var('dbt_artifacts_loader')['project'] %}
+{% set dataset = var('dbt_artifacts_loader')['dataset'] %}
+
+{{
+  config(
+    enabled=true,
+    full_refresh=none,
+    materialized="view",
+    database=project,
+    schema=dataset,
+    alias="generic_test_run_results_v4",
+    persist_docs={"relation": true, "columns": true},
+    labels={
+      "modeled_by": "dbt",
+      "app": "dbt-artifacts-loader",
+     },
+  )
+}}
+
+WITH run_results AS (
+  SELECT
+    run_results.*,
+    (SELECT AS STRUCT generic_tests.*) AS generic_test,
+  FROM {{ ref("expanded_run_results_v4") }} AS run_results
+  LEFT OUTER JOIN {{ ref("parsed_generic_test_node_v4") }} AS generic_tests
+    ON run_results.unique_id = generic_tests.unique_id
+      AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, generic_tests.metadata.generated_at, DAY))  <= 2
+  WHERE generic_tests.unique_id IS NOT NULL
+    AND timing_name IN ("execute")
+)
+-- Extract only run results whose metadata is the most close to that of model.
+, nearest_manifests AS (
+  SELECT
+    ROW_NUMBER() OVER (PARTITION BY unique_id, metadata.invocation_id ORDER BY generated_at_diff) AS rank,
+    * EXCEPT(generated_at_diff)
+  FROM (
+      SELECT
+        ABS(DATETIME_DIFF(metadata.generated_at, generic_test.metadata.generated_at, SECOND)) AS generated_at_diff,
+        *,
+      FROM run_results
+  )
+)
+
+SELECT
+  * EXCEPT(rank)
+FROM nearest_manifests
+WHERE rank = 1

dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/schema.yml

@@ -0,0 +1,29 @@
+---
+version: 2
+
+models:
+  - name: generic_test_run_results_v4
+    description: |
+      This is a denormalized schema test run results v4.
+
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - metadata.invocation_id
+            - unique_id
+            - timing_name
+
+    columns:
+      - name: metadata.invocation_id
+        description: "invocation ID"
+        tests:
+          - not_null
+      - name: unique_id
+        description: "unique test ID"
+        tests:
+          - not_null
+      - name: timing_name
+        description: "timing name"
+        tests:
+          - accepted_values:
+              values: ["execute"]

dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/last_generic_test_run_results_v4.sql

@@ -0,0 +1,30 @@
+{% set project = var('dbt_artifacts_loader')['project'] %}
+{% set dataset = var('dbt_artifacts_loader')['dataset'] %}
+
+{{
+  config(
+    enabled=true,
+    full_refresh=none,
+    materialized="view",
+    database=project,
+    schema=dataset,
+    alias="last_generic_test_run_results_v4",
+    persist_docs={"relation": true, "columns": true},
+    labels={
+      "modeled_by": "dbt",
+      "app": "dbt-artifacts-loader",
+     },
+  )
+}}
+
+WITH run_results AS (
+  SELECT
+    ROW_NUMBER() OVER (PARTITION BY unique_id ORDER BY completed_at DESC) AS rank,
+    *,
+  FROM {{ ref("generic_test_run_results_v4") }}
+)
+
+SELECT
+  * EXCEPT(rank)
+FROM run_results
+WHERE rank = 1

dbt_artifacts_models/models/marts/v4/run_results_v4/last_generic_test_run_results_v4/schema.yml

@@ -0,0 +1,30 @@
+---
+version: 2
+
+models:
+  - name: last_generic_test_run_results_v4
+    description: |
+      The table contains only the latest record of data test run results v4.
+
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - metadata.invocation_id
+            - unique_id
+            - timing_name
+
+    columns:
+      - name: metadata.invocation_id
+        description: "invocation ID"
+        tests:
+          - not_null
+      - name: unique_id
+        description: "unique test ID"
+        tests:
+          - not_null
+          - unique
+      - name: timing_name
+        description: "timing name"
+        tests:
+          - accepted_values:
+              values: ["execute"]

dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/last_model_run_results_v4.sql

@@ -0,0 +1,30 @@
+{% set project = var('dbt_artifacts_loader')['project'] %}
+{% set dataset = var('dbt_artifacts_loader')['dataset'] %}
+
+{{
+  config(
+    enabled=true,
+    full_refresh=none,
+    materialized="view",
+    database=project,
+    schema=dataset,
+    alias="last_model_run_results_v4",
+    persist_docs={"relation": true, "columns": true},
+    labels={
+      "modeled_by": "dbt",
+      "app": "dbt-artifacts-loader",
+     },
+  )
+}}
+
+WITH run_results AS (
+  SELECT
+    ROW_NUMBER() OVER (PARTITION BY unique_id ORDER BY completed_at DESC) AS rank,
+    *,
+  FROM {{ ref("model_run_results_v4") }}
+)
+
+SELECT
+  * EXCEPT(rank)
+FROM run_results
+WHERE rank = 1

dbt_artifacts_models/models/marts/v4/run_results_v4/last_model_run_results_v4/schema.yml

@@ -0,0 +1,30 @@
+---
+version: 2
+
+models:
+  - name: last_model_run_results_v4
+    description: |
+      The table contains only the latest record of model run results v4.
+
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - metadata.invocation_id
+            - unique_id
+            - timing_name
+
+    columns:
+      - name: metadata.invocation_id
+        description: "invocation ID"
+        tests:
+          - not_null
+      - name: unique_id
+        description: "unique test ID"
+        tests:
+          - not_null
+          - unique
+      - name: timing_name
+        description: "timing name"
+        tests:
+          - accepted_values:
+              values: ["execute"]

dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/last_singular_test_run_results_v4.sql

@@ -0,0 +1,30 @@
+{% set project = var('dbt_artifacts_loader')['project'] %}
+{% set dataset = var('dbt_artifacts_loader')['dataset'] %}
+
+{{
+  config(
+    enabled=true,
+    full_refresh=none,
+    materialized="view",
+    database=project,
+    schema=dataset,
+    alias="last_singular_test_run_results_v4",
+    persist_docs={"relation": true, "columns": true},
+    labels={
+      "modeled_by": "dbt",
+      "app": "dbt-artifacts-loader",
+     },
+  )
+}}
+
+WITH run_results AS (
+  SELECT
+    ROW_NUMBER() OVER (PARTITION BY unique_id ORDER BY completed_at DESC) AS rank,
+    *,
+  FROM {{ ref("singular_test_run_results_v4") }}
+)
+
+SELECT
+  * EXCEPT(rank)
+FROM run_results
+WHERE rank = 1

dbt_artifacts_models/models/marts/v4/run_results_v4/last_singular_test_run_results_v4/schema.yml

@@ -0,0 +1,30 @@
+---
+version: 2
+
+models:
+  - name: last_singular_test_run_results_v4
+    description: |
+      The table contains only the latest record of schema test run results v4.
+
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - metadata.invocation_id
+            - unique_id
+            - timing_name
+
+    columns:
+      - name: metadata.invocation_id
+        description: "invocation ID"
+        tests:
+          - not_null
+      - name: unique_id
+        description: "unique test ID"
+        tests:
+          - not_null
+          - unique
+      - name: timing_name
+        description: "timing name"
+        tests:
+          - accepted_values:
+              values: ["execute"]

dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/last_snapshot_run_results_v4.sql

@@ -0,0 +1,30 @@
+{% set project = var('dbt_artifacts_loader')['project'] %}
+{% set dataset = var('dbt_artifacts_loader')['dataset'] %}
+
+{{
+  config(
+    enabled=true,
+    full_refresh=none,
+    materialized="view",
+    database=project,
+    schema=dataset,
+    alias="last_snapshot_run_results_v4",
+    persist_docs={"relation": true, "columns": true},
+    labels={
+      "modeled_by": "dbt",
+      "app": "dbt-artifacts-loader",
+     },
+  )
+}}
+
+WITH run_results AS (
+  SELECT
+    ROW_NUMBER() OVER (PARTITION BY unique_id ORDER BY completed_at DESC) AS rank,
+    *,
+  FROM {{ ref("snapshot_run_results_v4") }}
+)
+
+SELECT
+  * EXCEPT(rank)
+FROM run_results
+WHERE rank = 1

dbt_artifacts_models/models/marts/v4/run_results_v4/last_snapshot_run_results_v4/schema.yml

@@ -0,0 +1,30 @@
+---
+version: 2
+
+models:
+  - name: last_snapshot_run_results_v4
+    description: |
+      The table contains only the latest record of snapshot run results v4.
+
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - metadata.invocation_id
+            - unique_id
+            - timing_name
+
+    columns:
+      - name: metadata.invocation_id
+        description: "invocation ID"
+        tests:
+          - not_null
+      - name: unique_id
+        description: "unique test ID"
+        tests:
+          - not_null
+          - unique
+      - name: timing_name
+        description: "timing name"
+        tests:
+          - accepted_values:
+              values: ["execute"]