diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb93d720..c52bc788 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,125 +15,30 @@ on: - main pull_request: workflow_dispatch: - inputs: - adapter: - description: The adapter to test against. Defaults to all supported adapters when blank. - type: string - required: false -env: - PYTHON_VERSION: "3.11" jobs: - determine-supported-adapters: - runs-on: ubuntu-latest - outputs: - adapters: ${{ steps.supported-adapters.outputs.adapters }} - steps: - - name: "Checkout ${{ github.event.repository }}" - uses: actions/checkout@v4 - - - name: "Set up Python ${{ env.PYTHON_VERSION }}" - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: "Install tox" - run: | - python -m pip install --upgrade pip - pip install tox - - - name: "Get list of supported adapters or use input adapter only" - id: list-adapters - run: | - if [ -z "${{ inputs.adapter }}" ]; then - # github adds a pip freeze and a new line we need to strip out - source supported_adapters.env - echo $SUPPORTED_ADAPTERS - echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT - else - echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT - fi - - - name: "Format adapter list for use as the matrix" - id: supported-adapters - run: | - # Convert to JSON array and output - supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")') - echo $supported_adapters - echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT - - - name: "[ANNOTATION] ${{ github.event.repository.name }} - Testing ${{ steps.supported-adapters.outputs.adapters }}" - run: | - title="${{ github.event.repository.name }} - adapters to test" - message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}" - echo "::notice $title::$message" - - run-tests: - runs-on: ubuntu-latest - needs: [determine-supported-adapters] - services: - postgres: - image: postgres - env: - POSTGRES_USER: ${{ vars.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASS }} - POSTGRES_DB: ${{ vars.POSTGRES_DATABASE }} - POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - strategy: - fail-fast: false - matrix: - adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}} - - steps: - - name: "Checkout ${{ github.event.repository }} " - uses: actions/checkout@v4 - - - name: "Set up Python ${{ env.PYTHON_VERSION }}" - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: "Install ${{ matrix.adapter }}" - run: | - python -m pip install --upgrade pip - pip install dbt-${{ matrix.adapter }} - - - name: "Install tox" - run: | - python -m pip install --upgrade pip - pip install tox - - - name: "Run integration tests with tox on ${{ matrix.adapter }}" - run: | - tox -e dbt_integration_${{ matrix.adapter }} - env: - POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} - POSTGRES_USER: ${{ vars.POSTGRES_USER }} - DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }} - POSTGRES_PORT: 5432 - POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }} - POSTGRES_SCHEMA: "dbt_utils_integration_tests_postgres_${{ github.run_number }}" - SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} - SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} - SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} - SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} - SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" - REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} - REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} - DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} - REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} - REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}" - REDSHIFT_PORT: 5439 - BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} - BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} - BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}" + run-tests: + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 + with: + # no need to pass postgres vars in. We can just use the defaults in the local container + # redshift + REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}" + REDSHIFT_PORT: 5439 + # bigquery + BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} + BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}" + # snowflake + SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} + SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" + secrets: + DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} + BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}