diff --git a/run_pyspark_examples.sh b/run_pyspark_examples.sh index a5cc7b8..3dcf805 100755 --- a/run_pyspark_examples.sh +++ b/run_pyspark_examples.sh @@ -30,6 +30,16 @@ export PYTHON_PATH=./environment/bin/python # Some hack for our json magic cat se*.json > spark_expectations_sample_rules.json +function check_fail () { + local ex="$1" + local code="$2" + if [ -f "${ex}.fail" ]; then + echo "ok"; + else + exit $code + fi +} + function run_example () { local ex="$1" # shellcheck disable=SC2046 @@ -45,7 +55,7 @@ function run_example () { --conf "spark.sql.catalog.local.warehouse=$PWD/warehouse" \ $(cat "${ex}.conf" || echo "") \ --name "${ex}" \ - "${ex}" 2>&1 | tee -a "${ex}.out" || echo "ok" + "${ex}" 2>&1 | tee -a "${ex}.out" || check_fail "$ex" $? } if [ $# -eq 1 ]; then diff --git a/se_complex.json b/se_complex.json index f958d69..8562d3a 100644 --- a/se_complex.json +++ b/se_complex.json @@ -1 +1,2 @@ -{"product_id": "pay", "table_name": "local.3rd_fake", "rule_type": "query_dq", "rule": "history", "column_name": "MaleBonusPercent", "expectation": "(select count(*) from 3rd_fake_view) < (select input_count from local.dq_stats WHERE table_name='local.3rd_fake')", "action_if_failed": "fail", "tag": "", "description": "We should always have more records than before", "enable_for_source_dq_validation": false, "enable_for_target_dq_validation": true, "is_active": true, "enable_error_drop_alert": true, "error_drop_threshold": 1} +{"product_id": "pay", "table_name": "local.3rd_fake", "rule_type": "row_dq", "rule": "bonus_checker", "column_name": "MaleBonusPercent", "expectation": "MaleBonusPercent > FemaleBonusPercent", "action_if_failed": "drop", "tag": "", "description": "Sample rule that the male bonuses should be higher. Thankfully this fails (but could be lower base pay etc.)", "enable_for_source_dq_validation": true, "enable_for_target_dq_validation": false, "is_active": true, "enable_error_drop_alert": true, "error_drop_threshold": 1} +{"product_id": "pay", "table_name": "local.3rd_fake", "rule_type": "query_dq", "rule": "history", "column_name": "MaleBonusPercent", "expectation": "(select count(*) from 3rd_fake_view) < (select input_count from local.dq_stats WHERE table_name='local.3rd_fake' LIMIT 1)", "action_if_failed": "fail", "tag": "", "description": "We should always have more records than before", "enable_for_source_dq_validation": false, "enable_for_target_dq_validation": true, "is_active": true, "enable_error_drop_alert": true, "error_drop_threshold": 1}