From 3594edfed3b03bdd4c9f9817815bba2bc03c59c6 Mon Sep 17 00:00:00 2001 From: nave91 Date: Mon, 6 Nov 2017 11:59:17 -0500 Subject: [PATCH 1/5] adding atleast_one and not_constant tests --- macros/schema_tests/atleast_one.sql | 27 +++++++++++++++++++++++++++ macros/schema_tests/not_constant.sql | 26 ++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 macros/schema_tests/atleast_one.sql create mode 100644 macros/schema_tests/not_constant.sql diff --git a/macros/schema_tests/atleast_one.sql b/macros/schema_tests/atleast_one.sql new file mode 100644 index 00000000..03dc3a31 --- /dev/null +++ b/macros/schema_tests/atleast_one.sql @@ -0,0 +1,27 @@ +{% macro test_atleast_one(model, arg) %} + +with validation as ( + + select + {{ arg }} as atleast_one_field + + from {{ model }} + +), + +validation_errors as ( + + + select + count(atleast_one_field) count_of_rows + + from validation +) + +-- returns null if there is atleast one non-null value +select count(*) + + from validation_errors + + where count_of_rows = 0 +{% endmacro %} \ No newline at end of file diff --git a/macros/schema_tests/not_constant.sql b/macros/schema_tests/not_constant.sql new file mode 100644 index 00000000..55334b36 --- /dev/null +++ b/macros/schema_tests/not_constant.sql @@ -0,0 +1,26 @@ +{% macro test_not_constant(model, arg) %} + +with validation as ( + + select + {{ arg }} as not_constant_field + + from {{ model }} + where {{ arg }} is not null + +), + +validation_errors as ( + + select + count(distinct not_constant_field) number_of_distinct_rows + + from validation + +) + +select count(*) +from validation_errors + where number_of_distinct_rows = 1 + +{% endmacro %} \ No newline at end of file From 44a710873e068daa8a6e5962f7326518019e1f82 Mon Sep 17 00:00:00 2001 From: nave91 Date: Mon, 6 Nov 2017 18:48:04 -0500 Subject: [PATCH 2/5] adding documentation and changing SQL to not use CTE --- README.md | 25 +++++++++++++++++++++ macros/schema_tests/atleast_one.sql | 28 +++++++++-------------- macros/schema_tests/not_constant.sql | 33 ++++++++++++---------------- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 9a7fd308..667e474d 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,31 @@ model_name: - ref('other_table_name') ``` + +#### atleast_one ([source](macros/schema_tests/atleast_one.sql)) +This schema test asserts if column has atleast one value. + +Usage: +``` +model_name: + constraints: + atleast_one: + - column_name + +``` + +#### not_constant ([source](macros/schema_tests/not_constant.sql)) +This schema test asserts if column does not have same value in all rows. + +Usage: +``` +model_name: + constraints: + not_constant: + - column_name + +``` + --- ### SQL helpers #### group_by ([source](macros/sql/groupby.sql)) diff --git a/macros/schema_tests/atleast_one.sql b/macros/schema_tests/atleast_one.sql index 03dc3a31..9965ab9d 100644 --- a/macros/schema_tests/atleast_one.sql +++ b/macros/schema_tests/atleast_one.sql @@ -1,27 +1,19 @@ {% macro test_atleast_one(model, arg) %} -with validation as ( - - select - {{ arg }} as atleast_one_field - - from {{ model }} - -), - -validation_errors as ( +select count(*) +from - select - count(atleast_one_field) count_of_rows + ( + select * - from validation -) + from ( + select + count({{ arg }}) count_of_rows --- returns null if there is atleast one non-null value -select count(*) + from {{ model }} + ) rows_with_values - from validation_errors + where count_of_rows = 0 ) validation_errors - where count_of_rows = 0 {% endmacro %} \ No newline at end of file diff --git a/macros/schema_tests/not_constant.sql b/macros/schema_tests/not_constant.sql index 55334b36..eb262d10 100644 --- a/macros/schema_tests/not_constant.sql +++ b/macros/schema_tests/not_constant.sql @@ -1,26 +1,21 @@ {% macro test_not_constant(model, arg) %} -with validation as ( - - select - {{ arg }} as not_constant_field - - from {{ model }} - where {{ arg }} is not null - -), - -validation_errors as ( +select count(*) +from + ( + select + count(distinct not_constant_field) count_of_distinct_rows + from ( + select + {{ arg }} as not_constant_field - select - count(distinct not_constant_field) number_of_distinct_rows + from {{ model }} - from validation + where {{ arg }} is not null -) + ) not_null_rows + ) discount_rows_count -select count(*) -from validation_errors - where number_of_distinct_rows = 1 + where count_of_distinct_rows = 1 -{% endmacro %} \ No newline at end of file +{% endmacro %} From b6301d2214a1b109b8d99ccb8c3696a597826b53 Mon Sep 17 00:00:00 2001 From: nave91 Date: Mon, 6 Nov 2017 22:20:15 -0500 Subject: [PATCH 3/5] using having instead of where sub query --- macros/schema_tests/atleast_one.sql | 18 +++++++----------- macros/schema_tests/not_constant.sql | 21 +++++++++------------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/macros/schema_tests/atleast_one.sql b/macros/schema_tests/atleast_one.sql index 9965ab9d..8166a39a 100644 --- a/macros/schema_tests/atleast_one.sql +++ b/macros/schema_tests/atleast_one.sql @@ -1,19 +1,15 @@ {% macro test_atleast_one(model, arg) %} select count(*) +from ( + select -from + count({{ arg }}) - ( - select * + from {{ model }} - from ( - select - count({{ arg }}) count_of_rows + having count({{ arg }}) = 0 - from {{ model }} - ) rows_with_values +) validation_errors - where count_of_rows = 0 ) validation_errors - -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/schema_tests/not_constant.sql b/macros/schema_tests/not_constant.sql index eb262d10..80b9b0df 100644 --- a/macros/schema_tests/not_constant.sql +++ b/macros/schema_tests/not_constant.sql @@ -1,21 +1,18 @@ + {% macro test_not_constant(model, arg) %} select count(*) -from - ( - select - count(distinct not_constant_field) count_of_distinct_rows - from ( - select - {{ arg }} as not_constant_field - from {{ model }} +from ( + + select + count(distinct {{ arg }}) + + from {{ model }} - where {{ arg }} is not null + having count(distinct {{ arg }}) = 1 - ) not_null_rows - ) discount_rows_count + ) validation_errors - where count_of_distinct_rows = 1 {% endmacro %} From 2a7677cadab57c0f4a219ef1b3bac9299d9b13bc Mon Sep 17 00:00:00 2001 From: nave91 Date: Mon, 6 Nov 2017 22:43:43 -0500 Subject: [PATCH 4/5] at least --- README.md | 6 +++--- macros/schema_tests/atleast_one.sql | 15 --------------- 2 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 macros/schema_tests/atleast_one.sql diff --git a/README.md b/README.md index 667e474d..0f5dfe17 100644 --- a/README.md +++ b/README.md @@ -73,14 +73,14 @@ model_name: ``` -#### atleast_one ([source](macros/schema_tests/atleast_one.sql)) -This schema test asserts if column has atleast one value. +#### at_least_one ([source](macros/schema_tests/at_least_one.sql)) +This schema test asserts if column has at least one value. Usage: ``` model_name: constraints: - atleast_one: + at_least_one: - column_name ``` diff --git a/macros/schema_tests/atleast_one.sql b/macros/schema_tests/atleast_one.sql deleted file mode 100644 index 8166a39a..00000000 --- a/macros/schema_tests/atleast_one.sql +++ /dev/null @@ -1,15 +0,0 @@ -{% macro test_atleast_one(model, arg) %} - -select count(*) -from ( - select - - count({{ arg }}) - - from {{ model }} - - having count({{ arg }}) = 0 - -) validation_errors - -{% endmacro %} From 895c3e6fff8289b334a54b04b396ab500e5975eb Mon Sep 17 00:00:00 2001 From: nave91 Date: Mon, 6 Nov 2017 22:43:50 -0500 Subject: [PATCH 5/5] at least --- macros/schema_tests/at_least_one.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 macros/schema_tests/at_least_one.sql diff --git a/macros/schema_tests/at_least_one.sql b/macros/schema_tests/at_least_one.sql new file mode 100644 index 00000000..94c00fe8 --- /dev/null +++ b/macros/schema_tests/at_least_one.sql @@ -0,0 +1,15 @@ +{% macro test_at_least_one(model, arg) %} + +select count(*) +from ( + select + + count({{ arg }}) + + from {{ model }} + + having count({{ arg }}) = 0 + +) validation_errors + +{% endmacro %}