From 41b3a124a481ed4f3d0952b528e97778973d961c Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Mon, 5 Feb 2024 17:39:18 -0300 Subject: [PATCH 1/2] Test remove --profiles-dir from generate docs --- .github/workflows/pull_request_build.yml | 29 +++++++++---------- .../account_usage/pipe_usage_history.yml | 1 + 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 5e967782..6aeb6db7 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -1,6 +1,6 @@ name: Test and Check on Pull Request -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy pull_request: paths: - transform/* @@ -38,27 +38,26 @@ jobs: DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform DATACOVES__YAML_DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/schedule - DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }} + DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }} - DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }} + DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }} - DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }} + DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }} DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }} - DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }} - DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} + DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }} + DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} # This is used by datacoves to drop the test database if permissions # cannot be applied when using the Datacoves permifrost security model. - DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }} - + DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }} # this is only here temporarilly until we updtae dbt-coves - DATACOVES__AIRFLOW_DAGS_YML_PATH: '' - DATACOVES__AIRFLOW_DAGS_PATH: '' - DATACOVES__AIRBYTE_HOST_NAME: '' - DATACOVES__AIRBYTE_PORT: '' + DATACOVES__AIRFLOW_DAGS_YML_PATH: "" + DATACOVES__AIRFLOW_DAGS_PATH: "" + DATACOVES__AIRBYTE_HOST_NAME: "" + DATACOVES__AIRBYTE_PORT: "" steps: - name: Checkout branch @@ -97,7 +96,7 @@ jobs: - name: Generate Docs Combining Prod and branch catalog.json #TODO: remove profiles-dir when dbt-coves is fixed - run: "dbt-coves generate docs --merge-deferred --state logs --profiles-dir /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt" + run: "dbt-coves generate docs --merge-deferred --state logs" - name: Run governance checks run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD" @@ -106,7 +105,7 @@ jobs: # most likely the schema was not set properly in dbt_project.yml so models built to default schema - name: Drop PR database on Failure to grant security access if: always() && (env.DATACOVES__DROP_DB_ON_FAIL == 'true') && (steps.grant-access-to-database.outcome == 'failure') - run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'" # yamllint disable-line rule:line-length + run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'" # yamllint disable-line rule:line-length airflow: name: Pull Request Airflow Tests diff --git a/transform/models/L1_inlets/account_usage/pipe_usage_history.yml b/transform/models/L1_inlets/account_usage/pipe_usage_history.yml index c5245770..7e8a2df4 100644 --- a/transform/models/L1_inlets/account_usage/pipe_usage_history.yml +++ b/transform/models/L1_inlets/account_usage/pipe_usage_history.yml @@ -6,6 +6,7 @@ models: meta: owner: "@alice" # Owner of the model model_maturity: in dev # Development status of the model + last_edit: Bruno columns: - name: pipe_id # Unique identifier for the pipe description: A unique identifier assigned to the pipe From 30ec70f11ce2328ab1eb4a8e0d783aec85ca0b24 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Mon, 5 Feb 2024 17:46:28 -0300 Subject: [PATCH 2/2] Add dbt debug config dir --- .github/workflows/pull_request_build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 6aeb6db7..ce9dfa9f 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -94,6 +94,10 @@ jobs: id: grant-access-to-database run: "dbt --no-write-json run-operation grant_access_to_pr_database" + - name: Get dbt profiles dir + #TODO: remove profiles-dir when dbt-coves is fixed + run: dbt debug --config-dir + - name: Generate Docs Combining Prod and branch catalog.json #TODO: remove profiles-dir when dbt-coves is fixed run: "dbt-coves generate docs --merge-deferred --state logs"