From ac48317e65288e3922f00409c02434b2ea04e359 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 25 Apr 2024 10:20:44 -0400 Subject: [PATCH 1/3] prefix env secrets with DBT_ENV_SECRET_ --- .github/workflows/integration.yml | 6 +++--- .github/workflows/release-internal.yml | 6 +++--- .github/workflows/release-prep.yml | 6 +++--- test.env.example | 6 +++--- tests/conftest.py | 14 +++++++------- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 94dece350..55b25c5c4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -66,9 +66,9 @@ jobs: DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} - DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} - DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} - DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} + DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} + DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} + DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 6334fd282..cf5a81546 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -89,9 +89,9 @@ jobs: DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} - DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} - DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} - DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} + DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} + DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} + DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml index 1a6e450c4..7925f9b5e 100644 --- a/.github/workflows/release-prep.yml +++ b/.github/workflows/release-prep.yml @@ -494,9 +494,9 @@ jobs: DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} - DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} - DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} - DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} + DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} + DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} + DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" diff --git a/test.env.example b/test.env.example index e69f700b7..f51fda161 100644 --- a/test.env.example +++ b/test.env.example @@ -1,11 +1,11 @@ # Cluster ID DBT_DATABRICKS_CLUSTER_NAME= # SQL Endpoint -DBT_DATABRICKS_ENDPOINT= +DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT= # Server Hostname value -DBT_DATABRICKS_HOST_NAME= +DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME= # personal token -DBT_DATABRICKS_TOKEN= +DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN= # file path to local ODBC driver ODBC_DRIVER= diff --git a/tests/conftest.py b/tests/conftest.py index efba41a5f..12c6b0234 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,9 +52,9 @@ def databricks_cluster_target(): return { "type": "spark", "method": "odbc", - "host": os.getenv("DBT_DATABRICKS_HOST_NAME"), + "host": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME"), "cluster": os.getenv("DBT_DATABRICKS_CLUSTER_NAME"), - "token": os.getenv("DBT_DATABRICKS_TOKEN"), + "token": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN"), "driver": os.getenv("ODBC_DRIVER"), "port": 443, "connect_retries": 3, @@ -68,9 +68,9 @@ def databricks_sql_endpoint_target(): return { "type": "spark", "method": "odbc", - "host": os.getenv("DBT_DATABRICKS_HOST_NAME"), - "endpoint": os.getenv("DBT_DATABRICKS_ENDPOINT"), - "token": os.getenv("DBT_DATABRICKS_TOKEN"), + "host": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME"), + "endpoint": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT"), + "token": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN"), "driver": os.getenv("ODBC_DRIVER"), "port": 443, "connect_retries": 3, @@ -82,9 +82,9 @@ def databricks_sql_endpoint_target(): def databricks_http_cluster_target(): return { "type": "spark", - "host": os.getenv("DBT_DATABRICKS_HOST_NAME"), + "host": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME"), "cluster": os.getenv("DBT_DATABRICKS_CLUSTER_NAME"), - "token": os.getenv("DBT_DATABRICKS_TOKEN"), + "token": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN"), "method": "http", "port": 443, "connect_retries": 3, From 3f9519319a55abde2e3fae22f1f939605dab73c1 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 25 Apr 2024 15:45:35 -0400 Subject: [PATCH 2/3] changie --- .changes/unreleased/Under the Hood-20240425-154527.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240425-154527.yaml diff --git a/.changes/unreleased/Under the Hood-20240425-154527.yaml b/.changes/unreleased/Under the Hood-20240425-154527.yaml new file mode 100644 index 000000000..b94141bbf --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240425-154527.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add secrets prefix for handling of secrets during testing +time: 2024-04-25T15:45:27.986709-04:00 +custom: + Author: mikealfare + Issue: "1025" From 045008633c2d33189de4b275fdb5c66140388585 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 26 Apr 2024 16:55:24 -0400 Subject: [PATCH 3/3] remove duplicative DBT_ prefix --- .github/workflows/integration.yml | 6 +++--- .github/workflows/release-internal.yml | 6 +++--- .github/workflows/release-prep.yml | 6 +++--- test.env.example | 6 +++--- tests/conftest.py | 14 +++++++------- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 55b25c5c4..69883ea6a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -66,9 +66,9 @@ jobs: DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} - DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} - DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} - DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} + DBT_ENV_SECRET_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} + DBT_ENV_SECRET_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} + DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index cf5a81546..50ba3bda8 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -89,9 +89,9 @@ jobs: DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} - DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} - DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} - DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} + DBT_ENV_SECRET_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} + DBT_ENV_SECRET_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} + DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml index 7925f9b5e..9e98504c0 100644 --- a/.github/workflows/release-prep.yml +++ b/.github/workflows/release-prep.yml @@ -494,9 +494,9 @@ jobs: DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} - DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} - DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} - DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} + DBT_ENV_SECRET_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} + DBT_ENV_SECRET_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} + DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" diff --git a/test.env.example b/test.env.example index f51fda161..02c8996e2 100644 --- a/test.env.example +++ b/test.env.example @@ -1,11 +1,11 @@ # Cluster ID DBT_DATABRICKS_CLUSTER_NAME= # SQL Endpoint -DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT= +DBT_ENV_SECRET_DATABRICKS_ENDPOINT= # Server Hostname value -DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME= +DBT_ENV_SECRET_DATABRICKS_HOST_NAME= # personal token -DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN= +DBT_ENV_SECRET_DATABRICKS_TOKEN= # file path to local ODBC driver ODBC_DRIVER= diff --git a/tests/conftest.py b/tests/conftest.py index 12c6b0234..7a0c8713c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,9 +52,9 @@ def databricks_cluster_target(): return { "type": "spark", "method": "odbc", - "host": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME"), + "host": os.getenv("DBT_ENV_SECRET_DATABRICKS_HOST_NAME"), "cluster": os.getenv("DBT_DATABRICKS_CLUSTER_NAME"), - "token": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN"), + "token": os.getenv("DBT_ENV_SECRET_DATABRICKS_TOKEN"), "driver": os.getenv("ODBC_DRIVER"), "port": 443, "connect_retries": 3, @@ -68,9 +68,9 @@ def databricks_sql_endpoint_target(): return { "type": "spark", "method": "odbc", - "host": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME"), - "endpoint": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_ENDPOINT"), - "token": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN"), + "host": os.getenv("DBT_ENV_SECRET_DATABRICKS_HOST_NAME"), + "endpoint": os.getenv("DBT_ENV_SECRET_DATABRICKS_ENDPOINT"), + "token": os.getenv("DBT_ENV_SECRET_DATABRICKS_TOKEN"), "driver": os.getenv("ODBC_DRIVER"), "port": 443, "connect_retries": 3, @@ -82,9 +82,9 @@ def databricks_sql_endpoint_target(): def databricks_http_cluster_target(): return { "type": "spark", - "host": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_HOST_NAME"), + "host": os.getenv("DBT_ENV_SECRET_DATABRICKS_HOST_NAME"), "cluster": os.getenv("DBT_DATABRICKS_CLUSTER_NAME"), - "token": os.getenv("DBT_ENV_SECRET_DBT_DATABRICKS_TOKEN"), + "token": os.getenv("DBT_ENV_SECRET_DATABRICKS_TOKEN"), "method": "http", "port": 443, "connect_retries": 3,