pr.yml

name: Test and Check on Pull Request

on:  # yamllint disable-line rule:truthy
  pull_request:
    paths:
      - transform/*
      - transform/**/*

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# This cancels a run if another change is pushed to the same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  dbt:
    name: Pull Request dbt Tests
    runs-on: ubuntu-latest
    # dbt 1.6 image
    container: datacoves/pi7-ci-basic-base:1.9.202309122155-20230923154521
    defaults:
      run:
        working-directory: /__w/balboa/balboa/transform

    env:
      DBT_PROFILES_DIR: /__w/balboa/balboa/automate/dbt
      DATACOVES__DBT_HOME: /__w/balboa/balboa/transform

      DATACOVES__MAIN__ACCOUNT: ${{ secrets.DATACOVES__MAIN__ACCOUNT }}
      DATACOVES__MAIN__ROLE: ${{ secrets.DATACOVES__MAIN__ROLE }}
      DATACOVES__MAIN__WAREHOUSE: ${{ secrets.DATACOVES__MAIN__WAREHOUSE }}
      DATACOVES__MAIN__DATABASE: BALBOA_PR_${{ github.event.number }}
      DATACOVES__MAIN__USER: ${{ secrets.DATACOVES__MAIN__USER }}
      DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }}

      DATACOVES__REPO_PATH: /__w/balboa/balboa
      DATACOVES__YAML_DAGS_FOLDER: /__w/balboa/balboa/schedule

    steps:
      - name: Checkout branch
        uses: actions/checkout@v3.5.0
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set Secure Directory
        run: git config --global --add safe.directory /__w/balboa/balboa

      - name: List of files changed
        run: "git diff origin/${{ github.event.pull_request.base.ref }} HEAD --name-status"

      - name: Install dbt packages
        run: "dbt deps"

      - name: Create PR database
        run: "dbt --no-write-json run-operation create_database"

      - name: Get prod manifest
        id: prod_manifest
        run: "../automate/dbt/get_artifacts.sh"

      - name: Run dbt build slim mode
        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }}
        run: "dbt build --fail-fast --defer --state logs --select state:modified+"

      - name: Run dbt build full run
        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }}
        run: "dbt build --fail-fast"

      - name: Grant access to PR database
        id: grant-access-to-database
        run: "dbt --no-write-json run-operation grant_access_to_pr_database"

      - name: Run governance checks
        run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"

      - name: Drop PR database on Failure
        if: always() && (steps.grant-access-to-database.outcome == 'failure')
        run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'"  # yamllint disable-line rule:line-length


  airflow:
    name: Pull Request Airflow Tests
    runs-on: ubuntu-latest
    # dbt 1.6 image
    container: datacoves/pi7-ci-airflow-base:1.9.202309122155-20230923154521
    env:
      AIRBYTE__EXTRACT_LOCATION: /__w/balboa/balboa/load
      AIRFLOW__CORE__DAGS_FOLDER: /__w/balboa/balboa/automate/airflow/dags
      AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 300

    steps:
      - name: Checkout branch
        uses: actions/checkout@v3.5.0
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Test DAG structure integrity (DagBag Loading)
        run: "python /usr/app/load_dagbag.py"

      - name: Test DBT Sources against DAGs' YAML files
        run: "python /usr/app/test_dags.py"