Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test remove --profiles-dir from generate docs #200

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 18 additions & 15 deletions .github/workflows/pull_request_build.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: Test and Check on Pull Request

on: # yamllint disable-line rule:truthy
on: # yamllint disable-line rule:truthy
pull_request:
paths:
- transform/*
Expand Down Expand Up @@ -38,27 +38,26 @@ jobs:
DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform
DATACOVES__YAML_DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/schedule

DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }}
DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }}

DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }}
DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }}
DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }}
DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }}

DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }}
DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }}
DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }}

DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }}
DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }}
DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }}
DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }}

# This is used by datacoves to drop the test database if permissions
# cannot be applied when using the Datacoves permifrost security model.
DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }}

DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }}

# this is only here temporarilly until we updtae dbt-coves
DATACOVES__AIRFLOW_DAGS_YML_PATH: ''
DATACOVES__AIRFLOW_DAGS_PATH: ''
DATACOVES__AIRBYTE_HOST_NAME: ''
DATACOVES__AIRBYTE_PORT: ''
DATACOVES__AIRFLOW_DAGS_YML_PATH: ""
DATACOVES__AIRFLOW_DAGS_PATH: ""
DATACOVES__AIRBYTE_HOST_NAME: ""
DATACOVES__AIRBYTE_PORT: ""

steps:
- name: Checkout branch
Expand Down Expand Up @@ -95,9 +94,13 @@ jobs:
id: grant-access-to-database
run: "dbt --no-write-json run-operation grant_access_to_pr_database"

- name: Get dbt profiles dir
#TODO: remove profiles-dir when dbt-coves is fixed
run: dbt debug --config-dir

- name: Generate Docs Combining Prod and branch catalog.json
#TODO: remove profiles-dir when dbt-coves is fixed
run: "dbt-coves generate docs --merge-deferred --state logs --profiles-dir /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt"
run: "dbt-coves generate docs --merge-deferred --state logs"

- name: Run governance checks
run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"
Expand All @@ -106,7 +109,7 @@ jobs:
# most likely the schema was not set properly in dbt_project.yml so models built to default schema
- name: Drop PR database on Failure to grant security access
if: always() && (env.DATACOVES__DROP_DB_ON_FAIL == 'true') && (steps.grant-access-to-database.outcome == 'failure')
run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'" # yamllint disable-line rule:line-length
run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'" # yamllint disable-line rule:line-length

airflow:
name: Pull Request Airflow Tests
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ models:
meta:
owner: "@alice" # Owner of the model
model_maturity: in dev # Development status of the model
last_edit: Bruno
columns:
- name: pipe_id # Unique identifier for the pipe
description: A unique identifier assigned to the pipe
Expand Down
Loading