Add a row dq rule to se complex so that the agg dq can execute.

run_pyspark_examples.sh

@@ -30,6 +30,16 @@ export PYTHON_PATH=./environment/bin/python
 # Some hack for our json magic
 cat se*.json > spark_expectations_sample_rules.json
 
+function check_fail () {
+  local ex="$1"
+  local code="$2"
+  if [ -f "${ex}.fail" ]; then
+    echo "ok";
+  else
+    exit $code
+  fi
+}
+
 function run_example () {
   local ex="$1"
   # shellcheck disable=SC2046
@@ -45,7 +55,7 @@ function run_example () {
 	       --conf "spark.sql.catalog.local.warehouse=$PWD/warehouse" \
 	       $(cat "${ex}.conf" || echo "") \
 	       --name "${ex}" \
-	       "${ex}" 2>&1 | tee -a "${ex}.out" || echo "ok"
+	       "${ex}" 2>&1 | tee -a "${ex}.out" || check_fail "$ex" $?
 }
 
 if [ $# -eq 1 ]; then

se_complex.json

@@ -1 +1,2 @@
-{"product_id": "pay", "table_name": "local.3rd_fake", "rule_type": "query_dq", "rule": "history", "column_name": "MaleBonusPercent", "expectation": "(select count(*) from 3rd_fake_view) < (select input_count from local.dq_stats WHERE table_name='local.3rd_fake')", "action_if_failed": "fail", "tag": "", "description": "We should always have more records than before", "enable_for_source_dq_validation": false, "enable_for_target_dq_validation": true, "is_active": true, "enable_error_drop_alert": true, "error_drop_threshold": 1}
+{"product_id": "pay", "table_name": "local.3rd_fake", "rule_type": "row_dq", "rule": "bonus_checker", "column_name": "MaleBonusPercent", "expectation": "MaleBonusPercent > FemaleBonusPercent", "action_if_failed": "drop", "tag": "", "description": "Sample rule that the male bonuses should be higher. Thankfully this fails (but could be lower base pay etc.)", "enable_for_source_dq_validation": true, "enable_for_target_dq_validation": false, "is_active": true, "enable_error_drop_alert": true, "error_drop_threshold": 1}
+{"product_id": "pay", "table_name": "local.3rd_fake", "rule_type": "query_dq", "rule": "history", "column_name": "MaleBonusPercent", "expectation": "(select count(*) from 3rd_fake_view) < (select input_count from local.dq_stats WHERE table_name='local.3rd_fake' LIMIT 1)", "action_if_failed": "fail", "tag": "", "description": "We should always have more records than before", "enable_for_source_dq_validation": false, "enable_for_target_dq_validation": true, "is_active": true, "enable_error_drop_alert": true, "error_drop_threshold": 1}