Skip to content

Commit

Permalink
Convert tox tests to central workflow (#947)
Browse files Browse the repository at this point in the history 
* use central workflow

* Update .github/workflows/ci.yml

* use tag
  • Loading branch information
@emmyoop
emmyoop authored Sep 26, 2024
1 parent e164c64 commit f13a392
Showing 1 changed file with 24 additions and 119 deletions.
143 changes: 24 additions & 119 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,125 +15,30 @@ on:
- main
pull_request:
workflow_dispatch:
inputs:
adapter:
description: The adapter to test against. Defaults to all supported adapters when blank.
type: string
required: false

env:
PYTHON_VERSION: "3.11"

jobs:
determine-supported-adapters:
runs-on: ubuntu-latest
outputs:
adapters: ${{ steps.supported-adapters.outputs.adapters }}
steps:
- name: "Checkout ${{ github.event.repository }}"
uses: actions/checkout@v4

- name: "Set up Python ${{ env.PYTHON_VERSION }}"
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: "Install tox"
run: |
python -m pip install --upgrade pip
pip install tox
- name: "Get list of supported adapters or use input adapter only"
id: list-adapters
run: |
if [ -z "${{ inputs.adapter }}" ]; then
# github adds a pip freeze and a new line we need to strip out
source supported_adapters.env
echo $SUPPORTED_ADAPTERS
echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT
else
echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT
fi
- name: "Format adapter list for use as the matrix"
id: supported-adapters
run: |
# Convert to JSON array and output
supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")')
echo $supported_adapters
echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT
- name: "[ANNOTATION] ${{ github.event.repository.name }} - Testing ${{ steps.supported-adapters.outputs.adapters }}"
run: |
title="${{ github.event.repository.name }} - adapters to test"
message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}"
echo "::notice $title::$message"
run-tests:
runs-on: ubuntu-latest
needs: [determine-supported-adapters]
services:
postgres:
image: postgres
env:
POSTGRES_USER: ${{ vars.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASS }}
POSTGRES_DB: ${{ vars.POSTGRES_DATABASE }}
POSTGRES_HOST: ${{ vars.POSTGRES_HOST }}
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
strategy:
fail-fast: false
matrix:
adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}}

steps:
- name: "Checkout ${{ github.event.repository }} "
uses: actions/checkout@v4

- name: "Set up Python ${{ env.PYTHON_VERSION }}"
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: "Install ${{ matrix.adapter }}"
run: |
python -m pip install --upgrade pip
pip install dbt-${{ matrix.adapter }}
- name: "Install tox"
run: |
python -m pip install --upgrade pip
pip install tox
- name: "Run integration tests with tox on ${{ matrix.adapter }}"
run: |
tox -e dbt_integration_${{ matrix.adapter }}
env:
POSTGRES_HOST: ${{ vars.POSTGRES_HOST }}
POSTGRES_USER: ${{ vars.POSTGRES_USER }}
DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }}
POSTGRES_PORT: 5432
POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }}
POSTGRES_SCHEMA: "dbt_utils_integration_tests_postgres_${{ github.run_number }}"
SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}"
REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}"
REDSHIFT_PORT: 5439
BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}"
run-tests:
uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
with:
# no need to pass postgres vars in. We can just use the defaults in the local container
# redshift
REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}"
REDSHIFT_PORT: 5439
# bigquery
BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}"
# snowflake
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}"
secrets:
DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}

0 comments on commit f13a392

Please sign in to comment.