From 464d41b5f9a6d64ab042bc976f67b6e6bc68428c Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Tue, 2 Apr 2024 08:36:16 -0500 Subject: [PATCH 1/3] add saved queries and exports to latest --- schemas/latest/dbt_yml_files-latest.json | 297 +++++++++-------------- tests/latest/invalid/dbt_yml_files.yml | 24 +- tests/latest/valid/dbt_yml_files.yml | 22 +- 3 files changed, 156 insertions(+), 187 deletions(-) diff --git a/schemas/latest/dbt_yml_files-latest.json b/schemas/latest/dbt_yml_files-latest.json index 14e7f1e..f4b2a1e 100644 --- a/schemas/latest/dbt_yml_files-latest.json +++ b/schemas/latest/dbt_yml_files-latest.json @@ -11,9 +11,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string" @@ -25,9 +23,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string" @@ -64,12 +60,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "depends_on", - "name", - "owner", - "type" - ], + "required": ["depends_on", "name", "owner", "type"], "$comment": "NB: depends_on is not strictly required, but is _expected_ according to the documentation", "properties": { "name": { @@ -80,13 +71,7 @@ }, "type": { "type": "string", - "enum": [ - "dashboard", - "notebook", - "analysis", - "ml", - "application" - ] + "enum": ["dashboard", "notebook", "analysis", "ml", "application"] }, "depends_on": { "type": "array", @@ -100,20 +85,14 @@ }, "maturity": { "type": "string", - "enum": [ - "high", - "medium", - "low" - ] + "enum": ["high", "medium", "low"] }, "meta": { "type": "object" }, "owner": { "type": "object", - "required": [ - "email" - ], + "required": ["email"], "properties": { "name": { "type": "string" @@ -137,10 +116,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name", - "owner" - ], + "required": ["name", "owner"], "properties": { "name": { "type": "string" @@ -166,9 +142,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string" @@ -180,9 +154,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string" @@ -208,12 +180,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name", - "label", - "type", - "type_params" - ], + "required": ["name", "label", "type", "type_params"], "properties": { "name": { "type": "string", @@ -254,9 +221,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string" @@ -266,11 +231,7 @@ }, "access": { "type": "string", - "enum": [ - "private", - "protected", - "public" - ] + "enum": ["private", "protected", "public"] }, "columns": { "type": "array", @@ -312,9 +273,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "v" - ], + "required": ["v"], "properties": { "columns": { "type": "array", @@ -342,13 +301,51 @@ "additionalProperties": false } }, + "saved_queries": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "description", "label", "query_params"], + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "exports": { + "type": "array", + "items": { + "$ref": "#/$defs/export" + } + }, + "label": { + "type": "string" + }, + "query_params": { + "type": "object", + "properties": { + "metrics": { + "$ref": "#/$defs/array_of_strings" + }, + "dimensions": { + "$ref": "#/$defs/array_of_strings" + }, + "where": { + "$ref": "#/$defs/array_of_strings" + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, "seeds": { "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string" @@ -414,10 +411,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name", - "model" - ], + "required": ["name", "model"], "properties": { "name": { "type": "string", @@ -468,9 +462,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string" @@ -560,9 +552,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "description": "How you will identify the schema in {{ source() }} calls. Unless `schema` is also set, this will be the name of the schema in the database.", @@ -623,9 +613,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "title": "Name", @@ -706,11 +694,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "name", - "model", - "expect" - ], + "required": ["name", "model", "expect"], "properties": { "name": { "type": "string" @@ -766,10 +750,7 @@ "format": { "description": "Defaults to `dict` when not specified", "type": "string", - "enum": [ - "dict", - "csv" - ] + "enum": ["dict", "csv"] }, "additionalProperties": false } @@ -812,10 +793,7 @@ "format": { "description": "Defaults to `dict` when not specified", "type": "string", - "enum": [ - "dict", - "csv" - ] + "enum": ["dict", "csv"] }, "input": { "description": "The relation whose inputs you need to mock. Enclose in ref or source without curly braces", @@ -833,9 +811,7 @@ "model": { "description": "The name of the model whose behaviour you are testing. Does not need to be wrapped in a ref.", "type": "string", - "examples": [ - "my_model" - ] + "examples": ["my_model"] }, "overrides": { "type": "object", @@ -899,9 +875,7 @@ }, "column_properties": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "uniqueItems": true, "properties": { "name": { @@ -952,9 +926,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "type" - ], + "required": ["type"], "properties": { "name": { "type": "string" @@ -988,10 +960,7 @@ "properties": { "relationships": { "type": "object", - "required": [ - "to", - "field" - ], + "required": ["to", "field"], "properties": { "name": { "type": "string" @@ -1026,9 +995,7 @@ "properties": { "accepted_values": { "type": "object", - "required": [ - "values" - ], + "required": ["values"], "properties": { "name": { "type": "string" @@ -1096,10 +1063,7 @@ }, "dimension": { "type": "object", - "required": [ - "name", - "type" - ], + "required": ["name", "type"], "anyOf": [ { "not": { @@ -1107,9 +1071,7 @@ } }, { - "required": [ - "type_params" - ] + "required": ["type_params"] } ], "properties": { @@ -1121,18 +1083,10 @@ "type": "string" }, "type": { - "enum": [ - "CATEGORICAL", - "TIME", - "categorical", - "time" - ] + "enum": ["CATEGORICAL", "TIME", "categorical", "time"] }, "expr": { - "type": [ - "string", - "boolean" - ] + "type": ["string", "boolean"] }, "is_partition": { "type": "boolean" @@ -1145,9 +1099,7 @@ }, "dimension_type_params": { "type": "object", - "required": [ - "time_granularity" - ], + "required": ["time_granularity"], "properties": { "time_granularity": { "enum": [ @@ -1188,10 +1140,7 @@ }, "entity": { "type": "object", - "required": [ - "name", - "type" - ], + "required": ["name", "type"], "properties": { "name": { "type": "string", @@ -1213,10 +1162,7 @@ "type": "string" }, "expr": { - "type": [ - "string", - "boolean" - ] + "type": ["string", "boolean"] }, "role": { "type": "string" @@ -1224,6 +1170,31 @@ }, "additionalProperties": false }, + "export": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + }, + "config": { + "type": "object", + "properties": { + "alias": { + "type": "string" + }, + "export_as": { + "enum": ["table", "view", "cache"] + }, + "schema": { + "type": "string" + } + }, + "addtionalProperties": true + } + }, + "additionalProperties": false + }, "freshness_definition": { "default": { "error_after": { @@ -1258,21 +1229,14 @@ }, "freshness_rules": { "type": "object", - "required": [ - "count", - "period" - ], + "required": ["count", "period"], "properties": { "count": { "$ref": "#/$defs/number_or_jinja_string" }, "period": { "type": "string", - "enum": [ - "minute", - "hour", - "day" - ] + "enum": ["minute", "hour", "day"] } }, "additionalProperties": false @@ -1302,15 +1266,10 @@ } }, "is-time-dimension": { - "required": [ - "type" - ], + "required": ["type"], "properties": { "type": { - "enum": [ - "TIME", - "time" - ] + "enum": ["TIME", "time"] } } }, @@ -1320,10 +1279,7 @@ }, "measure": { "type": "object", - "required": [ - "name", - "agg" - ], + "required": ["name", "agg"], "properties": { "name": { "type": "string", @@ -1368,11 +1324,7 @@ "type": "string" }, "expr": { - "type": [ - "string", - "integer", - "boolean" - ] + "type": ["string", "integer", "boolean"] }, "non_additive_dimension": { "$ref": "#/$defs/non_additive_dimension" @@ -1430,10 +1382,7 @@ "$ref": "#/$defs/metric_input_measure" }, "expr": { - "type": [ - "string", - "boolean" - ] + "type": ["string", "boolean"] }, "grain_to_date": { "type": "string" @@ -1482,10 +1431,7 @@ "type": "array", "items": { "type": "object", - "required": [ - "database", - "project" - ], + "required": ["database", "project"], "properties": { "database": { "type": "string" @@ -1529,20 +1475,11 @@ }, "on_configuration_change": { "type": "string", - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "on_schema_change": { "type": "string", - "enum": [ - "append_new_columns", - "fail", - "ignore", - "sync_all_columns" - ] + "enum": ["append_new_columns", "fail", "ignore", "sync_all_columns"] }, "snowflake_warehouse": { "type": "string" @@ -1558,20 +1495,13 @@ }, "non_additive_dimension": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], "properties": { "name": { "type": "string" }, "window_choice": { - "enum": [ - "MIN", - "MAX", - "min", - "max" - ] + "enum": ["MIN", "MAX", "min", "max"] }, "window_groupings": { "type": "array", @@ -1655,10 +1585,7 @@ }, { "type": "string", - "enum": [ - "warn", - "error" - ] + "enum": ["warn", "error"] } ] }, @@ -1686,4 +1613,4 @@ "additionalProperties": false } } -} \ No newline at end of file +} diff --git a/tests/latest/invalid/dbt_yml_files.yml b/tests/latest/invalid/dbt_yml_files.yml index c9d5579..309909f 100644 --- a/tests/latest/invalid/dbt_yml_files.yml +++ b/tests/latest/invalid/dbt_yml_files.yml @@ -72,4 +72,26 @@ unit_tests: expect: rows: | col_1,col_2 - val_1,val_2 \ No newline at end of file + val_1,val_2 + + +saved_queries: + - name: test_saved_query + description: + term: paper + label: Test saved query + query_params: + metrics: simple_metric + group_by: + - "Dimension('user__ds')" + where: + - "{{ Dimension('user__ds', 'DAY') }} <= now()" + - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'" + exports: + - name: my_export + config: + alias: my_export_alias + export_as: joel + schema: + - my_export_schema_name + - my_export_schema_name_2} \ No newline at end of file diff --git a/tests/latest/valid/dbt_yml_files.yml b/tests/latest/valid/dbt_yml_files.yml index bedadc1..c6e1815 100644 --- a/tests/latest/valid/dbt_yml_files.yml +++ b/tests/latest/valid/dbt_yml_files.yml @@ -189,4 +189,24 @@ metrics: type_params: measure: customers_with_orders filter: | - {{ Dimension('customer__customer_type') }} = 'new' \ No newline at end of file + {{ Dimension('customer__customer_type') }} = 'new' + + +saved_queries: + - name: test_saved_query + description: "{{ doc('saved_query_description') }}" + label: Test saved query + query_params: + metrics: + - simple_metric + group_by: + - "Dimension('user__ds')" + where: + - "{{ Dimension('user__ds', 'DAY') }} <= now()" + - "{{ Dimension('user__ds', 'DAY') }} >= '2023-01-01'" + exports: + - name: my_export + config: + alias: my_export_alias + export_as: table + schema: my_export_schema_name \ No newline at end of file From bcdd1ed388707f21b05edc5f6ffd17a02d0ee86b Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Tue, 2 Apr 2024 08:53:11 -0500 Subject: [PATCH 2/3] fix additional props --- schemas/latest/dbt_yml_files-latest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/latest/dbt_yml_files-latest.json b/schemas/latest/dbt_yml_files-latest.json index f4b2a1e..44bc501 100644 --- a/schemas/latest/dbt_yml_files-latest.json +++ b/schemas/latest/dbt_yml_files-latest.json @@ -1190,7 +1190,7 @@ "type": "string" } }, - "addtionalProperties": true + "additionalProperties": true } }, "additionalProperties": false From e444bf5b8167fa9d6a62f50750c4bfc2351b20c4 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 23 May 2024 14:16:08 -0500 Subject: [PATCH 3/3] yolo --- schemas/latest/dbt_yml_files-latest.json | 262 ++++++++++++++++++----- 1 file changed, 210 insertions(+), 52 deletions(-) diff --git a/schemas/latest/dbt_yml_files-latest.json b/schemas/latest/dbt_yml_files-latest.json index de65196..1032fd9 100644 --- a/schemas/latest/dbt_yml_files-latest.json +++ b/schemas/latest/dbt_yml_files-latest.json @@ -11,7 +11,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -23,7 +25,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -60,7 +64,12 @@ "type": "array", "items": { "type": "object", - "required": ["depends_on", "name", "owner", "type"], + "required": [ + "depends_on", + "name", + "owner", + "type" + ], "$comment": "NB: depends_on is not strictly required, but is _expected_ according to the documentation", "properties": { "name": { @@ -71,7 +80,13 @@ }, "type": { "type": "string", - "enum": ["dashboard", "notebook", "analysis", "ml", "application"] + "enum": [ + "dashboard", + "notebook", + "analysis", + "ml", + "application" + ] }, "depends_on": { "type": "array", @@ -85,14 +100,29 @@ }, "maturity": { "type": "string", - "enum": ["high", "medium", "low"] + "enum": [ + "high", + "medium", + "low" + ] }, "meta": { "type": "object" }, "owner": { "type": "object", - "anyOf": [{ "required": ["email"] }, { "required": ["name"] }], + "anyOf": [ + { + "required": [ + "email" + ] + }, + { + "required": [ + "name" + ] + } + ], "properties": { "name": { "type": "string" @@ -116,7 +146,10 @@ "type": "array", "items": { "type": "object", - "required": ["name", "owner"], + "required": [ + "name", + "owner" + ], "properties": { "name": { "type": "string" @@ -142,7 +175,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -154,7 +189,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -180,7 +217,12 @@ "type": "array", "items": { "type": "object", - "required": ["name", "label", "type", "type_params"], + "required": [ + "name", + "label", + "type", + "type_params" + ], "properties": { "name": { "type": "string", @@ -221,7 +263,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -231,7 +275,11 @@ }, "access": { "type": "string", - "enum": ["private", "protected", "public"] + "enum": [ + "private", + "protected", + "public" + ] }, "columns": { "type": "array", @@ -273,7 +321,9 @@ "type": "array", "items": { "type": "object", - "required": ["v"], + "required": [ + "v" + ], "properties": { "columns": { "type": "array", @@ -305,7 +355,12 @@ "type": "array", "items": { "type": "object", - "required": ["name", "description", "label", "query_params"], + "required": [ + "name", + "description", + "label", + "query_params" + ], "properties": { "name": { "type": "string" @@ -325,10 +380,10 @@ "query_params": { "type": "object", "properties": { - "metrics": { + "dimensions": { "$ref": "#/$defs/array_of_strings" }, - "dimensions": { + "metrics": { "$ref": "#/$defs/array_of_strings" }, "where": { @@ -345,7 +400,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -411,7 +468,10 @@ "type": "array", "items": { "type": "object", - "required": ["name", "model"], + "required": [ + "name", + "model" + ], "properties": { "name": { "type": "string", @@ -462,7 +522,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -552,7 +614,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "description": "How you will identify the schema in {{ source() }} calls. Unless `schema` is also set, this will be the name of the schema in the database.", @@ -613,7 +677,9 @@ "type": "array", "items": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "title": "Name", @@ -694,7 +760,11 @@ "type": "array", "items": { "type": "object", - "required": ["name", "model", "expect"], + "required": [ + "name", + "model", + "expect" + ], "properties": { "name": { "type": "string" @@ -750,7 +820,10 @@ "format": { "description": "Defaults to `dict` when not specified", "type": "string", - "enum": ["dict", "csv"] + "enum": [ + "dict", + "csv" + ] }, "additionalProperties": false } @@ -793,7 +866,10 @@ "format": { "description": "Defaults to `dict` when not specified", "type": "string", - "enum": ["dict", "csv"] + "enum": [ + "dict", + "csv" + ] }, "input": { "description": "The relation whose inputs you need to mock. Enclose in ref or source without curly braces", @@ -811,7 +887,9 @@ "model": { "description": "The name of the model whose behaviour you are testing. Does not need to be wrapped in a ref.", "type": "string", - "examples": ["my_model"] + "examples": [ + "my_model" + ] }, "overrides": { "type": "object", @@ -875,7 +953,9 @@ }, "column_properties": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "uniqueItems": true, "properties": { "name": { @@ -926,7 +1006,9 @@ "type": "array", "items": { "type": "object", - "required": ["type"], + "required": [ + "type" + ], "properties": { "name": { "type": "string" @@ -960,7 +1042,10 @@ "properties": { "relationships": { "type": "object", - "required": ["to", "field"], + "required": [ + "to", + "field" + ], "properties": { "name": { "type": "string" @@ -995,7 +1080,9 @@ "properties": { "accepted_values": { "type": "object", - "required": ["values"], + "required": [ + "values" + ], "properties": { "name": { "type": "string" @@ -1063,7 +1150,10 @@ }, "dimension": { "type": "object", - "required": ["name", "type"], + "required": [ + "name", + "type" + ], "anyOf": [ { "not": { @@ -1071,7 +1161,9 @@ } }, { - "required": ["type_params"] + "required": [ + "type_params" + ] } ], "properties": { @@ -1083,10 +1175,18 @@ "type": "string" }, "type": { - "enum": ["CATEGORICAL", "TIME", "categorical", "time"] + "enum": [ + "CATEGORICAL", + "TIME", + "categorical", + "time" + ] }, "expr": { - "type": ["string", "boolean"] + "type": [ + "string", + "boolean" + ] }, "is_partition": { "type": "boolean" @@ -1099,7 +1199,9 @@ }, "dimension_type_params": { "type": "object", - "required": ["time_granularity"], + "required": [ + "time_granularity" + ], "properties": { "time_granularity": { "enum": [ @@ -1140,7 +1242,10 @@ }, "entity": { "type": "object", - "required": ["name", "type"], + "required": [ + "name", + "type" + ], "properties": { "name": { "type": "string", @@ -1162,7 +1267,10 @@ "type": "string" }, "expr": { - "type": ["string", "boolean"] + "type": [ + "string", + "boolean" + ] }, "role": { "type": "string" @@ -1172,7 +1280,9 @@ }, "export": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" @@ -1184,7 +1294,11 @@ "type": "string" }, "export_as": { - "enum": ["table", "view", "cache"] + "enum": [ + "table", + "view", + "cache" + ] }, "schema": { "type": "string" @@ -1229,14 +1343,21 @@ }, "freshness_rules": { "type": "object", - "required": ["count", "period"], + "required": [ + "count", + "period" + ], "properties": { "count": { "$ref": "#/$defs/number_or_jinja_string" }, "period": { "type": "string", - "enum": ["minute", "hour", "day"] + "enum": [ + "minute", + "hour", + "day" + ] } }, "additionalProperties": false @@ -1266,10 +1387,15 @@ } }, "is-time-dimension": { - "required": ["type"], + "required": [ + "type" + ], "properties": { "type": { - "enum": ["TIME", "time"] + "enum": [ + "TIME", + "time" + ] } } }, @@ -1279,7 +1405,10 @@ }, "measure": { "type": "object", - "required": ["name", "agg"], + "required": [ + "name", + "agg" + ], "properties": { "name": { "type": "string", @@ -1324,7 +1453,11 @@ "type": "string" }, "expr": { - "type": ["string", "integer", "boolean"] + "type": [ + "string", + "integer", + "boolean" + ] }, "non_additive_dimension": { "$ref": "#/$defs/non_additive_dimension" @@ -1382,7 +1515,10 @@ "$ref": "#/$defs/metric_input_measure" }, "expr": { - "type": ["string", "boolean"] + "type": [ + "string", + "boolean" + ] }, "grain_to_date": { "type": "string" @@ -1431,7 +1567,10 @@ "type": "array", "items": { "type": "object", - "required": ["database", "project"], + "required": [ + "database", + "project" + ], "properties": { "database": { "type": "string" @@ -1475,11 +1614,20 @@ }, "on_configuration_change": { "type": "string", - "enum": ["apply", "continue", "fail"] + "enum": [ + "apply", + "continue", + "fail" + ] }, "on_schema_change": { "type": "string", - "enum": ["append_new_columns", "fail", "ignore", "sync_all_columns"] + "enum": [ + "append_new_columns", + "fail", + "ignore", + "sync_all_columns" + ] }, "snowflake_warehouse": { "type": "string" @@ -1495,13 +1643,20 @@ }, "non_additive_dimension": { "type": "object", - "required": ["name"], + "required": [ + "name" + ], "properties": { "name": { "type": "string" }, "window_choice": { - "enum": ["MIN", "MAX", "min", "max"] + "enum": [ + "MIN", + "MAX", + "min", + "max" + ] }, "window_groupings": { "type": "array", @@ -1585,7 +1740,10 @@ }, { "type": "string", - "enum": ["warn", "error"] + "enum": [ + "warn", + "error" + ] } ] }, @@ -1613,4 +1771,4 @@ "additionalProperties": false } } -} +} \ No newline at end of file