swap to copiler error, account for ignore columns

dbt-labs · May 22, 2023 · 6341a6f · 6341a6f
1 parent c23b946
commit 6341a6f
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 7 deletions.
diff --git a/integration_tests/models/generic_tests/schema.yml b/integration_tests/models/generic_tests/schema.yml
@@ -255,5 +255,5 @@ models:
     tests:
       - dbt_utils.equality:
           compare_model: ref('data_test_equality_a')
-          error_if: "<1" #sneaky way to ensure that the test is returning failing rows
-          warn_if: "<0"
+          ignore_columns:
+            - col_c
diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql
@@ -29,14 +29,44 @@
 {%- do dbt_utils._is_relation(model, 'test_equality') -%}
 
 {# Ensure there are no extra columns in the compare_model vs model #}
-{%- if not compare_columns and not ignore_columns -%}
+{%- if not compare_columns -%}
     {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%}
-    {%- set compare_columns_set = set(adapter.get_columns_in_relation(model) | map(attribute='quoted'))  -%}
-    {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%}
-    {%- set compare_model_columns_set = set(adapter.get_columns_in_relation(compare_model) | map(attribute='quoted')) -%}
+    {%- do dbt_utils._is_ephemeral(compare_model, 'test_equality') -%}
+
+    {%- set model_columns = adapter.get_columns_in_relation(model) -%}
+    {%- set compare_model_columns = adapter.get_columns_in_relation(compare_model) -%}
+
+
+    {%- if ignore_columns -%}
+        {#-- Lower case ignore columns for easier comparison --#}
+        {%- set ignore_columns = ignore_columns | map("lower") | list %}
+
+        {# Filter out the excluded columns #}
+        {%- set include_columns = [] %}
+        {%- set include_model_columns = [] %}
+        {%- for column in model_columns -%}
+            {%- if column.name | lower not in ignore_columns -%}
+                {% do include_columns.append(column) %}
+            {%- endif %}
+        {%- endfor %}
+        {%- for column in compare_model_columns -%}
+            {%- if column.name | lower not in ignore_columns -%}
+                {% do include_model_columns.append(column) %}
+            {%- endif %}
+        {%- endfor %}
+
+        {%- set compare_columns_set = set(include_columns | map(attribute='quoted')) %}
+        {%- set compare_model_columns_set = set(include_model_columns | map(attribute='quoted')) %}
+    {%- else -%}
+        {%- set compare_columns_set = set(model_columns | map(attribute='quoted')) %}
+        {%- set compare_model_columns_set = set(compare_model_columns | map(attribute='quoted')) %}
+    {%- endif -%}
+
     {% if compare_columns_set != compare_model_columns_set %}
-        {{ return("select cast('b_minus_a' as " ~ dbt.type_string() ~") as which_diff from " ~ compare_model) }}
+        {{ exceptions.raise_compiler_error(compare_model ~" has less columns than " ~ model ~ ", please ensure they have the same columns or use the `compare_columns` or `ignore_columns` arguments to subset them.") }}
     {% endif %}
+
+
 {% endif %}
 
 {%- if not precision -%}