add precision option to equality test

CHANGELOG.md

@@ -8,6 +8,12 @@
 ## Contributors:
 --->
 
+# Unreleased
+## New features
+- The `equality` test now accepts an additional argument, `precision` to aide in comparing floating point numbers ([#757](https://github.com/dbt-labs/dbt-utils/issues/757), [#765](https://github.com/dbt-labs/dbt-utils/pull/765))
+## Contributors: 
+- [@rlh1994](https://github.com/rlh1994)
+
 # Unreleased
 ## New features
 - Add option to ignore columns in equality test ([#734](https://github.com/dbt-labs/dbt-utils/issues/734), [#737](https://github.com/dbt-labs/dbt-utils/pull/737))

README.md

@@ -113,7 +113,7 @@ This test supports the `group_by_columns` parameter; see [Grouping in tests](#gr
 
 #### equality ([source](macros/generic_tests/equality.sql))
 
-Asserts the equality of two relations. Optionally specify a subset of columns to compare or ignore.
+Asserts the equality of two relations. Optionally specify a subset of columns to compare or ignore, and a precision to compare numeric columns on.
 
 **Usage:**
 
@@ -135,6 +135,7 @@ models:
           compare_columns:
             - first_column
             - second_column
+          precision: 4
 
   # compare all columns except the ones on the ignore list
   - name: model_name_ignore_columns

integration_tests/data/schema_tests/data_test_equality_floats.csv

@@ -0,0 +1,11 @@
+id,float_number
+05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888186
+cfae9054-940b-42a1-84d4-052daae6194f,81.2511656
+6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959675
+c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100841
+59caed0d-53d6-473c-a88c-3726c7693f05,68.6029434
+b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861425
+26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662223
+b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524680
+11c979b7-2661-4375-8143-7c9b54b90627,19.5755431
+a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237047

integration_tests/data/schema_tests/data_test_equality_floats_columns.csv

@@ -0,0 +1,11 @@
+id,float_number,to_ignore
+05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888186,a
+cfae9054-940b-42a1-84d4-052daae6194f,81.2511656,a
+6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959675,a
+c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100841,a
+59caed0d-53d6-473c-a88c-3726c7693f05,68.6029434,a
+b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861425,a
+26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662223,a
+b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524680,a
+11c979b7-2661-4375-8143-7c9b54b90627,19.5755431,a
+a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237047,a

integration_tests/dbt_project.yml

@@ -50,7 +50,7 @@ seeds:
     sql:
       data_events_20180103:
         +schema: events
-      
+
       data_get_column_values_dropped:
         # this.incorporate() to hardcode the node's type as otherwise dbt doesn't know it yet
         +post-hook: "{% do adapter.drop_relation(this.incorporate(type='table')) %}"
@@ -71,3 +71,11 @@ seeds:
       data_test_sequential_timestamps:
         +column_types:
           my_timestamp: timestamp
+
+      data_test_equality_floats:
+        +column_types:
+          float_number: float
+
+      data_test_equality_floats_columns:
+        +column_types:
+          float_number: float

integration_tests/models/generic_tests/schema.yml

@@ -89,14 +89,14 @@ seeds:
           upper_bound_column: valid_to
           partition_by: subscription_id
           zero_length_range_allowed: true
- 
+
   - name: data_unique_combination_of_columns
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - month
             - product
-  
+
   - name: data_cardinality_equality_a
     columns:
       - name: same_name
@@ -209,7 +209,41 @@ models:
             - first_name
             - last_name
             - email
-
+
+  - name: test_equality_floats
+    tests:
+      # test precision only
+      - dbt_utils.equality:
+          compare_model: ref('data_test_equality_floats')
+          precision: 4
+      - dbt_utils.equality:
+          compare_model: ref('data_test_equality_floats')
+          precision: 8
+          error_if: "<1" #sneaky way to ensure that the test is returning failing rows
+          warn_if: "<0"
+
+  - name: test_equality_floats_columns
+    tests:
+    # Positive assertion tests
+      - dbt_utils.equality:
+          compare_model: ref('data_test_equality_floats_columns')
+          compare_columns:
+            - id
+            - float_number
+          precision: 4
+      - dbt_utils.equality:
+          compare_model: ref('data_test_equality_floats_columns')
+          ignore_columns:
+            - to_ignore
+          precision: 4
+    # all columns should fail even with rounding
+      - dbt_utils.equality:
+          compare_model: ref('data_test_equality_floats_columns')
+          precision: 4
+          error_if: "<1" #sneaky way to ensure that the test is returning failing rows
+          warn_if: "<0"
+
+
   - name: test_fewer_rows_than
     tests:
       - dbt_utils.fewer_rows_than:

integration_tests/models/generic_tests/test_equality_floats.sql

@@ -0,0 +1,9 @@
+with data as (
+
+    select * from {{ ref('data_test_equality_floats') }}
+
+)
+
+select
+    id, float_number + 0.0000001 as float_number
+from data

integration_tests/models/generic_tests/test_equality_floats_columns.sql

@@ -0,0 +1,9 @@
+with data as (
+
+    select * from {{ ref('data_test_equality_floats') }}
+
+)
+
+select
+    id, float_number + 0.0000001 as float_number, 'b' as to_ignore
+from data

macros/generic_tests/equality.sql

@@ -1,14 +1,14 @@
-{% test equality(model, compare_model, compare_columns=None, ignore_columns=None) %}
-  {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns, ignore_columns)) }}
+{% test equality(model, compare_model, compare_columns=None, ignore_columns=None, precision = None) %}
+  {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns, ignore_columns, precision)) }}
 {% endtest %}
 
-{% macro default__test_equality(model, compare_model, compare_columns=None, ignore_columns=None) %}
+{% macro default__test_equality(model, compare_model, compare_columns=None, ignore_columns=None, precision = None) %}
 
 {%- if compare_columns and ignore_columns -%}
     {{ exceptions.raise_compiler_error("Both a compare and an ignore list were provided to the `equality` macro. Only one is allowed") }}
 {%- endif -%}
 
-{% set set_diff %}  
+{% set set_diff %}
     count(*) + coalesce(abs(
         sum(case when which_diff = 'a_minus_b' then 1 else 0 end) -
         sum(case when which_diff = 'b_minus_a' then 1 else 0 end)
@@ -26,32 +26,62 @@
 -- setup
 {%- do dbt_utils._is_relation(model, 'test_equality') -%}
 
-{#-
-If the compare_cols arg is provided, we can run this test without querying the
-information schema — this allows the model to be an ephemeral model
--#}
-
-{%- if not compare_columns -%}
+{%- if not precision -%}
+    {#-
+        If the compare_cols arg is provided, we can run this test without querying the
+        information schema — this allows the model to be an ephemeral model
+    -#}
+    {%- if not compare_columns -%}
+        {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%}
+        {%- set compare_columns = adapter.get_columns_in_relation(model)-%}
+
+
+        {%- if ignore_columns -%}
+            {#-- Lower case ignore columns for easier comparison --#}
+            {%- set ignore_columns = ignore_columns | map("lower") | list %}
+
+            {# Filter out the excluded columns #}
+            {%- set include_columns = [] %}
+            {%- for column in compare_columns -%}
+                {%- if column.name | lower not in ignore_columns -%}
+                    {% do include_columns.append(column) %}
+                {%- endif %}
+            {%- endfor %}
+
+            {%- set compare_columns = include_columns | map(attribute='quoted') %}
+        {%- else -%}
+            {%- set compare_columns = compare_columns | map(attribute='quoted') %}
+        {%- endif -%}
+    {%- endif -%}
+
+    {% set compare_cols_csv = compare_columns | join(', ') %}
+
+{% else %}
+    {#-
+        If rounding is required, we need to get the types, so it can't be ephermeral
+    -#}
     {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%}
-    {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='name') -%}
-{%- endif -%}
-
-{%- if ignore_columns -%}
-    {#-- Lower case ignore columns for easier comparison --#}
-    {%- set ignore_columns = ignore_columns | map("lower") | list %}
-
-    {%- set include_columns = [] %}
-    {%- for column in compare_columns -%}
-        {%- if column | lower not in ignore_columns -%}
-            {% do include_columns.append(column) %}
-        {%- endif %}
-    {%- endfor %}
+    {%- set columns = adapter.get_columns_in_relation(model) -%}
+
+    {% set columns_list = [] %}
+    {%- for col in columns -%}
+        {%- if (
+                (col.name|lower in compare_columns|map('lower') or not compare_columns) and
+                (col.name|lower not in ignore_columns|map('lower') or not ignore_columns)
+                ) -%}
+            {# Databricks double type is not picked up by any number type checks in dbt #}
+            {%- if col.is_float() or col.is_numeric() or col.data_type == 'double' -%}
+                {# Cast is required due to postgres not having round for a double precision number #}
+                {%- do columns_list.append('round(cast(' ~ col.quoted ~ ' as ' ~ dbt.type_numeric() ~ '),' ~ precision ~ ') as ' ~ col.quoted) -%}
+            {%- else -%}
+                {%- do columns_list.append(col.quoted) -%}
+            {%- endif -%}
+        {% endif %}
+    {%- endfor -%}
+
+    {% set compare_cols_csv = columns_list | join(', ') %}
 
-    {%- set compare_columns = include_columns %}
-
-{%- endif -%}
-
-{% set compare_cols_csv = compare_columns | join(', ') %}
+{% endif %}
 
 with a as (