diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 5e967782..ce9dfa9f 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -1,6 +1,6 @@ name: Test and Check on Pull Request -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy pull_request: paths: - transform/* @@ -38,27 +38,26 @@ jobs: DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform DATACOVES__YAML_DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/schedule - DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }} + DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }} - DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }} + DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }} - DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }} + DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }} DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }} - DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }} - DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} + DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }} + DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} # This is used by datacoves to drop the test database if permissions # cannot be applied when using the Datacoves permifrost security model. - DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }} - + DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }} # this is only here temporarilly until we updtae dbt-coves - DATACOVES__AIRFLOW_DAGS_YML_PATH: '' - DATACOVES__AIRFLOW_DAGS_PATH: '' - DATACOVES__AIRBYTE_HOST_NAME: '' - DATACOVES__AIRBYTE_PORT: '' + DATACOVES__AIRFLOW_DAGS_YML_PATH: "" + DATACOVES__AIRFLOW_DAGS_PATH: "" + DATACOVES__AIRBYTE_HOST_NAME: "" + DATACOVES__AIRBYTE_PORT: "" steps: - name: Checkout branch @@ -95,9 +94,13 @@ jobs: id: grant-access-to-database run: "dbt --no-write-json run-operation grant_access_to_pr_database" + - name: Get dbt profiles dir + #TODO: remove profiles-dir when dbt-coves is fixed + run: dbt debug --config-dir + - name: Generate Docs Combining Prod and branch catalog.json #TODO: remove profiles-dir when dbt-coves is fixed - run: "dbt-coves generate docs --merge-deferred --state logs --profiles-dir /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt" + run: "dbt-coves generate docs --merge-deferred --state logs" - name: Run governance checks run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD" @@ -106,7 +109,7 @@ jobs: # most likely the schema was not set properly in dbt_project.yml so models built to default schema - name: Drop PR database on Failure to grant security access if: always() && (env.DATACOVES__DROP_DB_ON_FAIL == 'true') && (steps.grant-access-to-database.outcome == 'failure') - run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'" # yamllint disable-line rule:line-length + run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'" # yamllint disable-line rule:line-length airflow: name: Pull Request Airflow Tests diff --git a/transform/models/L1_inlets/account_usage/pipe_usage_history.yml b/transform/models/L1_inlets/account_usage/pipe_usage_history.yml index c5245770..7e8a2df4 100644 --- a/transform/models/L1_inlets/account_usage/pipe_usage_history.yml +++ b/transform/models/L1_inlets/account_usage/pipe_usage_history.yml @@ -6,6 +6,7 @@ models: meta: owner: "@alice" # Owner of the model model_maturity: in dev # Development status of the model + last_edit: Bruno columns: - name: pipe_id # Unique identifier for the pipe description: A unique identifier assigned to the pipe