Skip to content

Commit

Permalink
Updates related to Airbyte Upgrade (#180)
Browse files Browse the repository at this point in the history 
* Updated configs for latest airbute version
* model updates for now airbyte version
* drop orphaned relations from prod database
* update GH Actions to use latest dbt 1.6 image
* fix linting error and update dbt version in pre-commit
* fix linting issues

---------

Co-authored-by: Noel Gomez <gomezn@convexa.ai>
  • Loading branch information
@noel
noel and Noel Gomez authored Sep 23, 2023
1 parent 99263bb commit f79bb41
Show file tree
Hide file tree
Showing 26 changed files with 335 additions and 222 deletions.
11 changes: 7 additions & 4 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ jobs:
build:
name: Deployment Script
runs-on: ubuntu-latest
# default container: datacoves/ci-basic-dbt-snowflake:1.6.202305082038-fa4492f2
container: datacoves/pi2-ci-basic-base:1.6.202305082038-20230521132745
# dbt 1.6 image
container: datacoves/pi7-ci-basic-base:1.9.202309122155-20230923154521
defaults:
run:
working-directory: /__w/balboa/balboa/transform
Expand Down Expand Up @@ -57,14 +57,17 @@ jobs:
MANIFEST_FOUND: ${{ steps.prod-manifest.outputs.manifest_found }}
run: "../automate/blue_green_run.py --ci-cd-run -t prd"

- name: Drop relations in db that are no longer in dbt
run: "dbt run-operation drop_orphanate_relations --args '{\"dry_run\": false}'"

- name: Delete merged test databases
run: "../automate/dbt/remove_test_databases.sh"

- name: Generate dbt documentation
run: "dbt docs generate"

- name: Optimize dbt documentation
run: "../automate/dbt/dbt_docs_optimizer.py"
# - name: Optimize dbt documentation
# run: "../automate/dbt/dbt_docs_optimizer.py"

- name: Deploy docs 🚀
uses: JamesIves/github-pages-deploy-action@4.1.4
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/pr.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ concurrency:

jobs:
dbt:
name: Pull Request Dbt Tests
name: Pull Request dbt Tests
runs-on: ubuntu-latest
# default container: datacoves/ci-basic-dbt-snowflake:1.6.202305082038-fa4492f2
container: datacoves/pi2-ci-basic-base:1.6.202305082038-20230521132745
# dbt 1.6 image
container: datacoves/pi7-ci-basic-base:1.9.202309122155-20230923154521
defaults:
run:
working-directory: /__w/balboa/balboa/transform
Expand Down Expand Up @@ -78,8 +78,8 @@ jobs:
airflow:
name: Pull Request Airflow Tests
runs-on: ubuntu-latest
#default container: datacoves/ci-airflow-dbt-snowflake:1.7.202307261721-dfc4e821
container: datacoves/pi2-ci-airflow-base:1.6.202305082038-20230521132745
# dbt 1.6 image
container: datacoves/pi7-ci-airflow-base:1.9.202309122155-20230923154521
env:
AIRBYTE__EXTRACT_LOCATION: /__w/balboa/balboa/load
AIRFLOW__CORE__DAGS_FOLDER: /__w/balboa/balboa/automate/airflow/dags
Expand Down
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,15 @@ repos:

- repo: https://github.com/sqlfluff/sqlfluff
# this is the version of sqlfluff, needs to be updated when using a new sqlfluff version (pip show sqlfluff)
rev: 2.0.3
rev: 2.3.2
hooks:
- id: sqlfluff-lint
language: python
# Need these two dependencies.
# sqlfluff-templater-dbt should match the version of sqlfluff above in rev (pip show sqlfluff-templater-dbt)
# dbt-snowflake needs to match the version in transform tab of Datacoves (pip show dbt-snowflake)
additional_dependencies:
["sqlfluff-templater-dbt==2.0.3", "dbt-snowflake==1.3.1"]
["sqlfluff-templater-dbt==2.3.2", "dbt-snowflake==1.6.2"]
args: [--config, transform/.sqlfluff]

- repo: https://github.com/adrienverge/yamllint.git
Expand Down
10 changes: 6 additions & 4 deletions load/airbyte/connections/country populations-datacoves snowflake.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "Country Populations <> Datacoves Snowflake",
"namespaceDefinition": "destination",
"namespaceDefinition": "source",
"namespaceFormat": "${SOURCE_NAMESPACE}",
"prefix": "",
"operationIds": [],
Expand Down Expand Up @@ -51,16 +51,18 @@
"destinationSyncMode": "overwrite",
"primaryKey": [],
"aliasName": "country_populations",
"selected": true
"selected": true,
"fieldSelectionEnabled": false
}
}
]
},
"scheduleType": "manual",
"status": "active",
"sourceCatalogId": "e4cb8570-994d-4c0a-95ab-da85e177a17d",
"sourceCatalogId": "b8999fc0-ac01-4e29-9893-0a0fabd677f8",
"geography": "auto",
"notifySchemaChanges": true,
"notifySchemaChanges": false,
"notifySchemaChangesByEmail": false,
"nonBreakingChangesPreference": "ignore",
"sourceName": "country populations",
"destinationName": "datacoves snowflake"
Expand Down
10 changes: 6 additions & 4 deletions load/airbyte/connections/golf-datacoves snowflake.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,17 +57,19 @@
"destinationSyncMode": "overwrite",
"primaryKey": [],
"aliasName": "golf",
"selected": true
"selected": true,
"fieldSelectionEnabled": false
}
}
]
},
"scheduleType": "manual",
"status": "active",
"sourceCatalogId": "18d1bd4c-b1c3-42f9-b44d-540034b1bf87",
"sourceCatalogId": "db485696-ba04-4b9c-9b15-b38b5ae54d37",
"geography": "auto",
"notifySchemaChanges": true,
"notifySchemaChanges": false,
"notifySchemaChangesByEmail": false,
"nonBreakingChangesPreference": "ignore",
"sourceName": "golf",
"destinationName": "datacoves snowflake"
}
}
68 changes: 35 additions & 33 deletions load/airbyte/connections/personal loans-datacoves snowflake.json
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"name": "Personal Loans <> Datacoves Snowflake",
"name": "Personal Loans \u2192 Datacoves Snowflake",
"namespaceDefinition": "customformat",
"namespaceFormat": "loans",
"prefix": "",
Expand All @@ -15,7 +15,7 @@
"properties": {
"DTI": {
"type": [
"string",
"number",
"null"
]
},
Expand Down Expand Up @@ -63,7 +63,7 @@
},
"PUB_REC": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -75,13 +75,13 @@
},
"INT_RATE": {
"type": [
"string",
"number",
"null"
]
},
"OPEN_ACC": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -99,25 +99,25 @@
},
"LOAN_AMNT": {
"type": [
"string",
"number",
"null"
]
},
"MEMBER_ID": {
"type": [
"string",
"number",
"null"
]
},
"OUT_PRNCP": {
"type": [
"string",
"number",
"null"
]
},
"REVOL_BAL": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -129,7 +129,7 @@
},
"TOTAL_ACC": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -141,7 +141,7 @@
},
"ANNUAL_INC": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -159,7 +159,7 @@
},
"RECOVERIES": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -171,19 +171,19 @@
},
"DELINQ_2YRS": {
"type": [
"string",
"number",
"null"
]
},
"FUNDED_AMNT": {
"type": [
"string",
"number",
"null"
]
},
"INSTALLMENT": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -195,13 +195,13 @@
},
"POLICY_CODE": {
"type": [
"string",
"number",
"null"
]
},
"TOTAL_PYMNT": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -219,13 +219,13 @@
},
"OUT_PRNCP_INV": {
"type": [
"string",
"number",
"null"
]
},
"TOTAL_REC_INT": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -237,31 +237,31 @@
},
"INQ_LAST_6MTHS": {
"type": [
"string",
"number",
"null"
]
},
"FUNDED_AMNT_INV": {
"type": [
"string",
"number",
"null"
]
},
"LAST_PYMNT_AMNT": {
"type": [
"string",
"number",
"null"
]
},
"TOTAL_PYMNT_INV": {
"type": [
"string",
"number",
"null"
]
},
"TOTAL_REC_PRNCP": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -279,7 +279,7 @@
},
"TOTAL_REC_LATE_FEE": {
"type": [
"string",
"number",
"null"
]
},
Expand All @@ -297,31 +297,31 @@
},
"MTHS_SINCE_LAST_DELINQ": {
"type": [
"string",
"number",
"null"
]
},
"MTHS_SINCE_LAST_RECORD": {
"type": [
"string",
"number",
"null"
]
},
"COLLECTION_RECOVERY_FEE": {
"type": [
"string",
"number",
"null"
]
},
"COLLECTIONS_12_MTHS_EX_MED": {
"type": [
"string",
"number",
"null"
]
},
"MTHS_SINCE_LAST_MAJOR_DEROG": {
"type": [
"string",
"number",
"null"
]
}
Expand All @@ -339,16 +339,18 @@
"destinationSyncMode": "overwrite",
"primaryKey": [],
"aliasName": "personal_loans",
"selected": true
"selected": true,
"fieldSelectionEnabled": false
}
}
]
},
"scheduleType": "manual",
"status": "active",
"sourceCatalogId": "e111be3a-eeb2-4597-8cd5-cd4c71d5b40a",
"sourceCatalogId": "9e00b943-1aae-4b9a-bf37-5be252c6ef8c",
"geography": "auto",
"notifySchemaChanges": true,
"notifySchemaChanges": false,
"notifySchemaChangesByEmail": false,
"nonBreakingChangesPreference": "ignore",
"sourceName": "personal loans",
"destinationName": "datacoves snowflake"
Expand Down
Loading

0 comments on commit f79bb41

Please sign in to comment.