Add dbt versions #16
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Perform deployment to production | |
| on: # yamllint disable-line rule:truthy | |
| pull_request: | |
| types: | |
| - closed | |
| branches: | |
| - main | |
| paths: | |
| - .github/workflows/* | |
| - automate/* | |
| - transform/* | |
| - transform/**/* | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| jobs: | |
| # Perform the deployment to Prod | |
| build: | |
| # Need to make sure that when the PR was closed, it was actually merged. | |
| if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' | |
| name: Deployment Script | |
| runs-on: ubuntu-latest | |
| # Set environment variables in | |
| # https://github.com//<your org>/<your repo>/settings/variables/actions | |
| # | |
| # Alternatively, You can define multiple ENV for different workflows. | |
| # https://github.com/<org>/<repo>/settings/environments | |
| # environment: PR_ENV | |
| container: datacoves/ci-basic-dbt-snowflake:3.2 | |
| defaults: | |
| run: | |
| working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| env: | |
| DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt | |
| DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }} | |
| DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }} | |
| DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }} | |
| DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }} | |
| DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }} | |
| DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }} | |
| DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} | |
| # This is used by datacoves to drop the staging database for blue/green | |
| # deployments, most likely you don't want to set this, we use it for demos | |
| DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }} | |
| steps: | |
| - name: Checkout branch | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.push.head.sha }} | |
| fetch-depth: 0 | |
| - name: Set Secure Directory | |
| run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| - name: Install dbt packages | |
| run: "dbt deps" | |
| - name: Get prod manifest | |
| id: prod-manifest | |
| run: "../automate/dbt/get_artifacts.sh" | |
| # this is needed bec our sample dynamic table is not getting refreshed | |
| - name: Refresh dynamic tables before blue-green | |
| run: "dbt run -s materialization config.materialized:dynamic_table --full-refresh" | |
| - name: Run blue / green deployment | |
| if: contains(github.event.pull_request.labels.*.name, 'full-refresh') != true | |
| id: run-blue-green | |
| env: | |
| MANIFEST_FOUND: ${{ steps.prod-manifest.outputs.manifest_found }} | |
| run: "dbt-coves blue-green --defer" | |
| - name: Run blue / green deployment with full-refresh | |
| if: contains(github.event.pull_request.labels.*.name, 'full-refresh') | |
| id: run-blue-green-full-refresh | |
| env: | |
| MANIFEST_FOUND: ${{ steps.prod-manifest.outputs.manifest_found }} | |
| run: "dbt-coves blue-green --defer --full-refresh" | |
| - name: Drop orphaned relations in db that are no longer in dbt | |
| run: "dbt run-operation drop_orphaned_relations --args '{\"dry_run\": false}'" | |
| - name: Generate dbt documentation | |
| run: "dbt docs generate" | |
| - name: Deploy docs 🚀 | |
| uses: JamesIves/github-pages-deploy-action@4.1.4 | |
| with: | |
| branch: dbt-docs | |
| folder: transform/target | |
| - name: Deploy Streamlit App | |
| run: "dbt run-operation deploy_streamlit_app --args \"{app_path: '../visualize/streamlit/loans-example', app_file: loans.py}\"" | |
| - name: Upload dbt artifacts | |
| run: "dbt run-operation upload_artifacts" | |
| - uses: fregante/setup-git-user@v2 | |
| - name: Bump dbt project and git project version | |
| run: "../automate/dbt/bump_dbt_project.sh" | |
| - name: Push version changes | |
| uses: ad-m/github-push-action@v0.6.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ github.ref_name }} | |
| tags: true | |
| - name: Drop staging db on Failure | |
| if: always() && (steps.run-blue-green.outcome == 'failure') | |
| run: "dbt --no-write-json run-operation drop_staging_db --args 'db_name: ${{env.DATACOVES__STAGING__DATABASE}}'" # yamllint disable-line rule:line-length | |
| # Drops the temporary PR database | |
| drop-pr-db-on-close: | |
| name: Drop PR Database on Close | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| # Set environment variables in | |
| # https://github.com//<your org>/<your repo>/settings/variables/actions | |
| # | |
| # Alternatively, You can define multiple ENV for different workflows. | |
| # https://github.com/<org>/<repo>/settings/environments | |
| # environment: PR_ENV | |
| container: datacoves/ci-basic-dbt-snowflake:3.2 | |
| defaults: | |
| run: | |
| working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| env: | |
| DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt | |
| DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform | |
| DATACOVES__YAML_DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/schedule | |
| DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }} | |
| DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }} | |
| DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }} | |
| DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }} | |
| DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }} | |
| DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }} | |
| DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }} | |
| steps: | |
| - name: Checkout branch | |
| uses: actions/checkout@v3.5.0 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set Secure Directory | |
| run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} | |
| - name: Install dbt packages | |
| run: "dbt deps" | |
| - name: Drop PR database | |
| run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'" # yamllint disable-line rule:line-length |