diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b76e40b..057d9c83 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,6 @@ jobs: POSTGRES_TEST_PASS: '' POSTGRES_TEST_PORT: 5432 POSTGRES_TEST_DBNAME: circle_test - POSTGRES_SCHEMA: dbt_utils_integration_tests_postgres steps: - checkout diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..68db8b10 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,127 @@ +# **what?** +# Run tests for dbt-utils against supported adapters + +# **why?** +# To ensure that dbt-utils works as expected with all supported adapters + +# **when?** +# On every PR, and every push to main and when manually triggered + +name: Package Integration Tests + +on: + push: + branches: + - main + pull_request_target: + workflow_dispatch: + inputs: + adapter: + description: The adapter to test against. Defaults to all supported adapters when blank. + type: string + required: false + +env: + PYTHON_VERSION: "3.11" + +jobs: + determine-supported-adapters: + runs-on: ubuntu-latest + outputs: + adapters: ${{ steps.supported-adapters.outputs.adapters }} + steps: + - name: "Checkout ${{ github.event.repository }}" + uses: actions/checkout@v4 + + - name: "Set up Python ${{ env.PYTHON_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: "Install tox" + run: | + python -m pip install --upgrade pip + pip install tox + + - name: "Get list of supported adapters or use input adapter only" + id: list-adapters + run: | + if [ -z "${{ inputs.adapter }}" ]; then + # github adds a pip freeze and a new line we need to strip out + source supported_adapters.env + echo $SUPPORTED_ADAPTERS + echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT + else + echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT + fi + + - name: "Format adapter list for use as the matrix" + id: supported-adapters + run: | + # Convert to JSON array and output + supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")') + echo $supported_adapters + echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT + + - name: "[ANNOTATION] ${{ github.event.repository.name }} - adapters to test" + run: | + title="${{ github.event.repository.name }} - adapters to test" + message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}" + echo "::notice $title::$message" + + run-tests: + runs-on: ubuntu-latest + needs: [determine-supported-adapters] + strategy: + fail-fast: false + matrix: + adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}} + + steps: + - name: "Checkout ${{ github.event.repository }} " + uses: actions/checkout@v4 + + - name: "Set up Python ${{ env.PYTHON_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: "Install ${{ matrix.adapter }}" + run: | + python -m pip install --upgrade pip + pip install dbt-${{ matrix.adapter }} + + - name: "Install tox" + run: | + python -m pip install --upgrade pip + pip install tox + + - name: "Run integration tests with tox on ${{ matrix.adapter }}" + run: | + tox -e dbt_integration_${{ matrix.adapter }} + env: + POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }} + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }} + POSTGRES_SCHEMA: "dbt_utils_integration_tests_postgres" + # snowflake env vars + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} + DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} + SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake" + # redshift + REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} + REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift" + REDSHIFT_PORT: 5439 + # bigquery + BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} + DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_SERVICE_ACCOUNT_JSON }} + BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index edffde0f..f37e918e 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -2,6 +2,8 @@ # HEY! This file is used in the dbt-utils integrations tests with CircleCI. # You should __NEVER__ check credentials into version control. Thanks for reading :) +# TODO: This file can be removed when CirceCI is no longer used for dbt-utils integration tests + integration_tests: target: postgres outputs: diff --git a/supported_adapters.env b/supported_adapters.env new file mode 100755 index 00000000..3acc8a8b --- /dev/null +++ b/supported_adapters.env @@ -0,0 +1 @@ +SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index e81e0810..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,3 +0,0 @@ -from pathlib import Path - -DBT_PROJECT_PATH = Path(__file__).parent.parent / "dbt_project" \ No newline at end of file diff --git a/tests/functional/test_example.py b/tests/functional/test_example.py deleted file mode 100644 index 96de9b22..00000000 --- a/tests/functional/test_example.py +++ /dev/null @@ -1,7 +0,0 @@ - - -# def test_failure(): -# assert 1 == 5 - -def test_pass(): - assert 1 == 1 \ No newline at end of file diff --git a/tox.ini b/tox.ini index e49c1ebe..c09e65dd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,6 @@ [tox] skipsdist = True envlist = lint_all, testenv -# this needs to be defined on a single line. tox has problems with multiline lists -# This list determined which adapters will be run by dbt centralized testing -supported_adapters = postgres, snowflake, redshift, bigquery [testenv] passenv = @@ -34,11 +31,6 @@ passenv = DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON BIGQUERY_SCHEMA -# This is required to ensure that the correct adapters are tested -[testenv:list_supported_adapters] -allowlist_externals = echo -commands = echo {[tox]supported_adapters} - # Uses pytest to run tests, very basic and doesn't require dbt [testenv:dbt_integration_pytest] changedir = {toxinidir}