add github workflow, move adapters

.circleci/config.yml

@@ -15,7 +15,6 @@ jobs:
       POSTGRES_TEST_PASS: ''
       POSTGRES_TEST_PORT: 5432
       POSTGRES_TEST_DBNAME: circle_test
-      POSTGRES_SCHEMA: dbt_utils_integration_tests_postgres
 
     steps:
       - checkout

.github/workflows/ci.yml

@@ -0,0 +1,127 @@
+# **what?**
+# Run tests for dbt-utils against supported adapters
+
+# **why?**
+# To ensure that dbt-utils works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to main and when manually triggered
+
+name: Package Integration Tests
+
+on:
+    push:
+        branches:
+            - main
+    pull_request_target:
+    workflow_dispatch:
+        inputs:
+            adapter:
+                description: The adapter to test against.  Defaults to all supported adapters when blank.
+                type: string
+                required: false
+
+env:
+    PYTHON_VERSION: "3.11"
+
+jobs:
+    determine-supported-adapters:
+        runs-on: ubuntu-latest
+        outputs:
+            adapters: ${{ steps.supported-adapters.outputs.adapters }}
+        steps:
+            - name: "Checkout ${{ github.event.repository }}"
+              uses: actions/checkout@v4
+
+            - name: "Set up Python ${{ env.PYTHON_VERSION }}"
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ env.PYTHON_VERSION }}
+
+            - name: "Install tox"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install tox
+
+            - name: "Get list of supported adapters or use input adapter only"
+              id: list-adapters
+              run: |
+                  if [ -z "${{ inputs.adapter }}" ]; then
+                      # github adds a pip freeze and a new line we need to strip out
+                      source supported_adapters.env
+                      echo $SUPPORTED_ADAPTERS
+                      echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT
+                  else
+                      echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT
+                  fi
+
+            - name: "Format adapter list for use as the matrix"
+              id: supported-adapters
+              run: |
+                  # Convert to JSON array and output
+                  supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")')
+                  echo $supported_adapters
+                  echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT
+
+            - name: "[ANNOTATION] ${{ github.event.repository.name }} - adapters to test"
+              run: |
+                  title="${{ github.event.repository.name }} - adapters to test"
+                  message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}"
+                  echo "::notice $title::$message"
+
+    run-tests:
+        runs-on: ubuntu-latest
+        needs: [determine-supported-adapters]
+        strategy:
+            fail-fast: false
+            matrix:
+                adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}}
+
+        steps:
+            - name: "Checkout ${{ github.event.repository }} "
+              uses: actions/checkout@v4
+
+            - name: "Set up Python ${{ env.PYTHON_VERSION }}"
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ env.PYTHON_VERSION }}
+
+            - name: "Install ${{ matrix.adapter }}"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install dbt-${{ matrix.adapter }}
+
+            - name: "Install tox"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install tox
+
+            - name: "Run integration tests with tox on ${{ matrix.adapter }}"
+              run: |
+                  tox -e dbt_integration_${{ matrix.adapter }}
+              env:
+                  POSTGRES_HOST: ${{ vars.POSTGRES_HOST }}
+                  POSTGRES_USER: ${{ vars.POSTGRES_USER }}
+                  DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }}
+                  POSTGRES_PORT: 5432
+                  POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }}
+                  POSTGRES_SCHEMA: "dbt_utils_integration_tests_postgres"
+                  # snowflake env vars
+                  SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+                  SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
+                  DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
+                  SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
+                  SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
+                  SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
+                  SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake"
+                  # redshift
+                  REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
+                  REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
+                  DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
+                  REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
+                  REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift"
+                  REDSHIFT_PORT: 5439
+                  # bigquery
+                  BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
+                  DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_SERVICE_ACCOUNT_JSON }}
+                  BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery"

integration_tests/ci/sample.profiles.yml

@@ -2,6 +2,8 @@
 # HEY! This file is used in the dbt-utils integrations tests with CircleCI.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
+# TODO: This file can be removed when CirceCI is no longer used for dbt-utils integration tests
+
 integration_tests:
   target: postgres
   outputs:

supported_adapters.env

@@ -0,0 +1 @@
+SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery

tox.ini

@@ -1,9 +1,6 @@
 [tox]
 skipsdist = True
 envlist = lint_all, testenv
-# this needs to be defined on a single line.  tox has problems with multiline lists
-# This list determined which adapters will be run by dbt centralized testing
-supported_adapters = postgres, snowflake, redshift, bigquery
 
 [testenv]
 passenv =
@@ -34,11 +31,6 @@ passenv =
     DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON
     BIGQUERY_SCHEMA
 
-# This is required to ensure that the correct adapters are tested
-[testenv:list_supported_adapters]
-allowlist_externals = echo
-commands = echo {[tox]supported_adapters}
-
 # Uses pytest to run tests, very basic and doesn't require dbt
 [testenv:dbt_integration_pytest]
 changedir = {toxinidir}