check for env var to drop demo dbs (#198)

* check for env vars to drop db on fail

---------

Co-authored-by: Noel Gomez <gomezn@convexa.ai>

.github/workflows/merge_to_main.yml

@@ -1,4 +1,4 @@
-name: Generate Docs on merge to Main
+name: Perform deployment to production
 
 on:  # yamllint disable-line rule:truthy
   pull_request:
@@ -53,6 +53,10 @@ jobs:
       DATACOVES__MAIN__USER:      ${{ vars.DATACOVES__MAIN__USER }}
       DATACOVES__MAIN__PASSWORD:  ${{ secrets.DATACOVES__MAIN__PASSWORD }}
 
+      # This is used by datacoves to drop the staging database for blue/green
+      # deployments, most likely you don't want to set this, we use it for demos
+      DATACOVES__DROP_DB_ON_FAIL:  ${{ vars.DATACOVES__DROP_DB_ON_FAIL }}
+
     steps:
       - name: Checkout branch
         uses: actions/checkout@v2

.github/workflows/pull_request_build.yml

@@ -47,6 +47,9 @@ jobs:
       DATACOVES__MAIN__USER:      ${{ vars.DATACOVES__MAIN__USER }}
       DATACOVES__MAIN__PASSWORD:  ${{ secrets.DATACOVES__MAIN__PASSWORD }}
 
+      # This is used by datacoves to drop the test database if permissions
+      # cannot be applied when using the Datacoves permifrost security model.
+      DATACOVES__DROP_DB_ON_FAIL:  ${{ vars.DATACOVES__DROP_DB_ON_FAIL }}
 
     steps:
       - name: Checkout branch
@@ -90,11 +93,12 @@ jobs:
       - name: Run governance checks
         run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"
 
-      - name: Drop PR database on Failure
-        if: always() && (steps.grant-access-to-database.outcome == 'failure')
+      # We drop the database when there is a failure to grant access to the db because
+      # most likely the schema was not set properly in dbt_project.yml so models built to default schema
+      - name: Drop PR database on Failure to grant security access
+        if: always() && (env.DATACOVES__DROP_DB_ON_FAIL == 'true') && (steps.grant-access-to-database.outcome == 'failure')
         run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'"  # yamllint disable-line rule:line-length
 
-
   airflow:
     name: Pull Request Airflow Tests
     runs-on: ubuntu-latest

automate/blue_green_run.py

@@ -20,6 +20,12 @@
 
 DBT_COVES__CLONE_PATH = tempfile.NamedTemporaryFile().name
 
+
+# This is used by datacoves to drop the staging database for blue/green
+# deployments, most likely you don't want to set this, we use it for demos
+DATACOVES__DROP_DB_ON_FAIL = os.environ.get("DATACOVES__DROP_DB_ON_FAIL", 'false')
+
+
 def main(is_ci_cd_run: bool = False, selector: str = None, target: str = None, full_refresh: bool = False):
     """
     Manages blue/green deployment workflow
@@ -34,9 +40,10 @@ def main(is_ci_cd_run: bool = False, selector: str = None, target: str = None, f
         dbt_target = f" -t {target}"
 
     ######
-    logging.info("\n\n===== REMOVE THIS IN A PRODUCTION DEPLOYMENT=====\n")
-    STAGING_DB_ARGS = '{"db_name": "' + DBT_STAGING_DB_NAME + '"}'
-    run_command(f'dbt-coves dbt -- run-operation drop_staging_db --args "{STAGING_DB_ARGS}" {dbt_target}')
+    if DATACOVES__DROP_DB_ON_FAIL == 'true':
+        logging.info("\n\n===== DROPPING STAGING DATABASE =====\n")
+        STAGING_DB_ARGS = '{"db_name": "' + DBT_STAGING_DB_NAME + '"}'
+        run_command(f'dbt-coves dbt -- run-operation drop_staging_db --args "{STAGING_DB_ARGS}" {dbt_target}')
     ######
 
     logging.info("Checking that staging database does not exist")

transform/models/L1_inlets/loans/personal_loans.yml

@@ -8,8 +8,6 @@ models:
         description: The state in which the borrower resides
         meta:
           masking_policy: masking_policy_pii_string
-        tests:
-          - not_null
       - name: annual_inc
         description: The borrower's annual income
         meta: