Updates related to Airbyte Upgrade #534
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test and Check on Pull Request | |
| on: # yamllint disable-line rule:truthy | |
| pull_request: | |
| paths: | |
| - transform/* | |
| - transform/**/* | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| # This cancels a run if another change is pushed to the same branch | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| dbt: | |
| name: Pull Request dbt Tests | |
| runs-on: ubuntu-latest | |
| # dbt 1.6 image | |
| container: datacoves/pi7-ci-basic-base:1.9.202309122155-20230923154521 | |
| defaults: | |
| run: | |
| working-directory: /__w/balboa/balboa/transform | |
| env: | |
| DBT_PROFILES_DIR: /__w/balboa/balboa/automate/dbt | |
| DATACOVES__DBT_HOME: /__w/balboa/balboa/transform | |
| DATACOVES__MAIN__ACCOUNT: ${{ secrets.DATACOVES__MAIN__ACCOUNT }} | |
| DATACOVES__MAIN__ROLE: ${{ secrets.DATACOVES__MAIN__ROLE }} | |
| DATACOVES__MAIN__WAREHOUSE: ${{ secrets.DATACOVES__MAIN__WAREHOUSE }} | |
| DATACOVES__MAIN__DATABASE: BALBOA_PR_${{ github.event.number }} | |
| DATACOVES__MAIN__USER: ${{ secrets.DATACOVES__MAIN__USER }} | |
| DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} | |
| DATACOVES__REPO_PATH: /__w/balboa/balboa | |
| DATACOVES__YAML_DAGS_FOLDER: /__w/balboa/balboa/schedule | |
| steps: | |
| - name: Checkout branch | |
| uses: actions/checkout@v3.5.0 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set Secure Directory | |
| run: git config --global --add safe.directory /__w/balboa/balboa | |
| - name: List of files changed | |
| run: "git diff origin/${{ github.event.pull_request.base.ref }} HEAD --name-status" | |
| - name: Install dbt packages | |
| run: "dbt deps" | |
| - name: Create PR database | |
| run: "dbt --no-write-json run-operation create_database" | |
| - name: Get prod manifest | |
| id: prod_manifest | |
| run: "../automate/dbt/get_artifacts.sh" | |
| - name: Run dbt build slim mode | |
| if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} | |
| run: "dbt build --fail-fast --defer --state logs --select state:modified+" | |
| - name: Run dbt build full run | |
| if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} | |
| run: "dbt build --fail-fast" | |
| - name: Grant access to PR database | |
| run: "dbt --no-write-json run-operation grant_access_to_pr_database" | |
| - name: Run governance checks | |
| run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD" | |
| airflow: | |
| name: Pull Request Airflow Tests | |
| runs-on: ubuntu-latest | |
| # dbt 1.6 image | |
| container: datacoves/pi7-ci-airflow-base:1.9.202309122155-20230923154521 | |
| env: | |
| AIRBYTE__EXTRACT_LOCATION: /__w/balboa/balboa/load | |
| AIRFLOW__CORE__DAGS_FOLDER: /__w/balboa/balboa/automate/airflow/dags | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 300 | |
| steps: | |
| - name: Checkout branch | |
| uses: actions/checkout@v3.5.0 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Test DAG structure integrity (DagBag Loading) | |
| run: "python /usr/app/load_dagbag.py" | |
| - name: Test DBT Sources against DAGs' YAML files | |
| run: "python /usr/app/test_dags.py" |