From df882a26ff722776a168e786bca679319262458e Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 23 Dec 2022 11:55:26 -0800 Subject: [PATCH] 1.0 update version-bump GHA (#563) * 1.0 update version-bump GHA * align dev-requirements naming w core * Bumping version to 1.0.2rc0 and generate changelog (#566) * Bumping version to 1.0.2rc0 and generate CHANGELOG * Update CHANGELOG.md Co-authored-by: Github Build Bot Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> * Bumping version to 1.0.2 and generate changelog (#569) * Bumping version to 1.0.2 and generate CHANGELOG * Update CHANGELOG.md Co-authored-by: Github Build Bot Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> * skip databricks integration tests * experimental skipping of integ tests Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot --- .bumpversion.cfg | 2 +- .changes/1.0.2.md | 6 + .../unreleased/Features-20220810-133356.yaml | 7 -- .../Under the Hood-20221209-143353.yaml | 7 -- .circleci/config.yml | 66 +++++++---- .github/workflows/release.yml | 6 +- .github/workflows/version-bump.yml | 103 +++--------------- CHANGELOG.md | 9 +- dbt/adapters/spark/__version__.py | 2 +- dev_requirements.txt => dev-requirements.txt | 0 setup.py | 2 +- tox.ini | 12 +- 12 files changed, 87 insertions(+), 135 deletions(-) create mode 100644 .changes/1.0.2.md delete mode 100644 .changes/unreleased/Features-20220810-133356.yaml delete mode 100644 .changes/unreleased/Under the Hood-20221209-143353.yaml rename dev_requirements.txt => dev-requirements.txt (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e1b5dc8b1..c8d5270fe 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.1 +current_version = 1.0.2 parse = (?P\d+) \.(?P\d+) \.(?P\d+) diff --git a/.changes/1.0.2.md b/.changes/1.0.2.md new file mode 100644 index 000000000..f23be9a0a --- /dev/null +++ b/.changes/1.0.2.md @@ -0,0 +1,6 @@ +## dbt-spark 1.0.2 - December 20, 2022 +### Features +- backport changie to 1.0.latest ([#417](https://github.com/dbt-labs/dbt-spark/issues/417), [#426](https://github.com/dbt-labs/dbt-spark/pull/426)) +### Under the Hood +- fix post new release tox issues around passenv and allowlist_externals ([#547](https://github.com/dbt-labs/dbt-spark/issues/547), [#547](https://github.com/dbt-labs/dbt-spark/pull/547)) + diff --git a/.changes/unreleased/Features-20220810-133356.yaml b/.changes/unreleased/Features-20220810-133356.yaml deleted file mode 100644 index e9468a15f..000000000 --- a/.changes/unreleased/Features-20220810-133356.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Features -body: backport changie to 1.0.latest -time: 2022-08-10T13:33:56.992461-05:00 -custom: - Author: mcknight-42 - Issue: "417" - PR: "426" diff --git a/.changes/unreleased/Under the Hood-20221209-143353.yaml b/.changes/unreleased/Under the Hood-20221209-143353.yaml deleted file mode 100644 index 5589a0f2e..000000000 --- a/.changes/unreleased/Under the Hood-20221209-143353.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Under the Hood -body: fix post new release tox issues around passenv and allowlist_externals -time: 2022-12-09T14:33:53.134371-06:00 -custom: - Author: McKnight-42 - Issue: "547" - PR: "547" diff --git a/.circleci/config.yml b/.circleci/config.yml index 4921fac98..53c1849d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,24 @@ jobs: - checkout - run: tox -e flake8,unit + integration-spark-session: + environment: + DBT_INVOCATION_ENV: circle + docker: + - image: godatadriven/pyspark:3.1 + steps: + - checkout + - run: apt-get update + - run: python3 -m pip install --upgrade pip + - run: apt-get install -y git gcc g++ unixodbc-dev libsasl2-dev + - run: python3 -m pip install tox + - run: + name: Run integration tests + command: tox -e integration-spark-session + no_output_timeout: 1h + - store_artifacts: + path: ./logs + integration-spark-thrift: environment: DBT_INVOCATION_ENV: circle @@ -61,21 +79,28 @@ jobs: integration-spark-databricks-http: environment: DBT_INVOCATION_ENV: circle + DBT_DATABRICKS_RETRY_ALL: True + DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" + DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" + DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com" docker: - image: fishtownanalytics/test-container:10 steps: - checkout - - run: - name: Run integration tests - command: tox -e integration-spark-databricks-http - no_output_timeout: 1h - - store_artifacts: - path: ./logs + # - run: + # name: Run integration tests + # command: tox -e integration-spark-databricks-http + # no_output_timeout: 1h + # - store_artifacts: + # path: ./logs integration-spark-databricks-odbc-cluster: &databricks-odbc environment: DBT_INVOCATION_ENV: circle ODBC_DRIVER: Simba # TODO: move env var to Docker image + DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" + DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" + DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com" docker: # image based on `fishtownanalytics/test-container` w/ Simba ODBC Spark driver installed - image: 828731156495.dkr.ecr.us-east-1.amazonaws.com/dbt-spark-odbc-test-container:latest @@ -84,29 +109,32 @@ jobs: aws_secret_access_key: $AWS_SECRET_ACCESS_KEY_STAGING steps: - checkout - - run: - name: Run integration tests - command: tox -e integration-spark-databricks-odbc-cluster - no_output_timeout: 1h - - store_artifacts: - path: ./logs - + # - run: + # name: Run integration tests + # command: tox -e integration-spark-databricks-odbc-cluster + # no_output_timeout: 1h + # - store_artifacts: + # path: ./logs + integration-spark-databricks-odbc-endpoint: <<: *databricks-odbc steps: - checkout - - run: - name: Run integration tests - command: tox -e integration-spark-databricks-odbc-sql-endpoint - no_output_timeout: 1h - - store_artifacts: - path: ./logs + # - run: + # name: Run integration tests + # command: tox -e integration-spark-databricks-odbc-sql-endpoint + # no_output_timeout: 1h + # - store_artifacts: + # path: ./logs workflows: version: 2 test-everything: jobs: - unit + - integration-spark-session: + requires: + - unit - integration-spark-thrift: requires: - unit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05e0c7a3b..884e3ebc3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: python3 -m venv env source env/bin/activate sudo apt-get install libsasl2-dev - pip install -r dev_requirements.txt + pip install -r dev-requirements.txt pip install twine wheel setuptools python setup.py sdist bdist_wheel pip install dist/dbt-spark-*.tar.gz @@ -48,7 +48,7 @@ jobs: python3 -m venv env source env/bin/activate sudo apt-get install libsasl2-dev - pip install -r dev_requirements.txt + pip install -r dev-requirements.txt bumpversion --config-file .bumpversion.cfg patch --new-version ${{env.version_number}} --allow-dirty git status @@ -111,7 +111,7 @@ jobs: python3 -m venv env source env/bin/activate sudo apt-get install libsasl2-dev - pip install -r dev_requirements.txt + pip install -r dev-requirements.txt pip install twine wheel setuptools python setup.py sdist bdist_wheel twine upload --non-interactive dist/dbt_spark-${{env.version_number}}-py3-none-any.whl dist/dbt-spark-${{env.version_number}}.tar.gz diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index 7fb8bb6eb..bde34d683 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -1,18 +1,15 @@ # **what?** -# This workflow will take a version number and a dry run flag. With that -# it will run versionbump to update the version number everywhere in the -# code base and then generate an update Docker requirements file. If this -# is a dry run, a draft PR will open with the changes. If this isn't a dry -# run, the changes will be committed to the branch this is run on. +# This workflow will take the new version number to bump to. With that +# it will run versionbump to update the version number everywhere in the +# code base and then run changie to create the corresponding changelog. +# A PR will be created with the changes that can be reviewed before committing. # **why?** -# This is to aid in releasing dbt and making sure we have updated -# the versions and Docker requirements in all places. +# This is to aid in releasing dbt and making sure we have updated +# the version in all places and generated the changelog. # **when?** -# This is triggered either manually OR -# from the repository_dispatch event "version-bump" which is sent from -# the dbt-release repo Action +# This is triggered manually name: Version Bump @@ -20,84 +17,12 @@ on: workflow_dispatch: inputs: version_number: - description: 'The version number to bump to' + description: 'The version number to bump to (ex. 1.2.0, 1.3.0b1)' required: true - is_dry_run: - description: 'Creates a draft PR to allow testing instead of committing to a branch' - required: true - default: 'true' - repository_dispatch: - types: [version-bump] - -jobs: - bump: - runs-on: ubuntu-latest - steps: - - name: Check out the repository - uses: actions/checkout@v2 - - - name: Set version and dry run values - id: variables - env: - VERSION_NUMBER: "${{ github.event.client_payload.version_number == '' && github.event.inputs.version_number || github.event.client_payload.version_number }}" - IS_DRY_RUN: "${{ github.event.client_payload.is_dry_run == '' && github.event.inputs.is_dry_run || github.event.client_payload.is_dry_run }}" - run: | - echo Repository dispatch event version: ${{ github.event.client_payload.version_number }} - echo Repository dispatch event dry run: ${{ github.event.client_payload.is_dry_run }} - echo Workflow dispatch event version: ${{ github.event.inputs.version_number }} - echo Workflow dispatch event dry run: ${{ github.event.inputs.is_dry_run }} - echo ::set-output name=VERSION_NUMBER::$VERSION_NUMBER - echo ::set-output name=IS_DRY_RUN::$IS_DRY_RUN - - - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - - name: Install python dependencies - run: | - sudo apt-get install libsasl2-dev - python3 -m venv env - source env/bin/activate - pip install --upgrade pip - - - name: Create PR branch - if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }} - run: | - git checkout -b bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID - git push origin bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID - git branch --set-upstream-to=origin/bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID - - - name: Bumping version - run: | - source env/bin/activate - pip install -r dev_requirements.txt - env/bin/bumpversion --allow-dirty --new-version ${{steps.variables.outputs.VERSION_NUMBER}} major - git status - - - name: Commit version bump directly - uses: EndBug/add-and-commit@v7 - if: ${{ steps.variables.outputs.IS_DRY_RUN == 'false' }} - with: - author_name: 'Github Build Bot' - author_email: 'buildbot@fishtownanalytics.com' - message: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}' - - - name: Commit version bump to branch - uses: EndBug/add-and-commit@v7 - if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }} - with: - author_name: 'Github Build Bot' - author_email: 'buildbot@fishtownanalytics.com' - message: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}' - branch: 'bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}' - push: 'origin origin/bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}' - - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 - if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }} - with: - author: 'Github Build Bot ' - draft: true - base: ${{github.ref}} - title: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}' - branch: 'bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}' +jobs: + version_bump_and_changie: + uses: dbt-labs/actions/.github/workflows/version-bump.yml@main + with: + version_number: ${{ inputs.version_number }} + secrets: inherit # ok since what we are calling is internally maintained diff --git a/CHANGELOG.md b/CHANGELOG.md index 147293f9e..dba48b950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ - Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-spark 1.0.2 - December 20, 2022 +### Features +- backport changie to 1.0.latest ([#417](https://github.com/dbt-labs/dbt-spark/issues/417), [#426](https://github.com/dbt-labs/dbt-spark/pull/426)) +### Under the Hood +- fix post new release tox issues around passenv and allowlist_externals ([#547](https://github.com/dbt-labs/dbt-spark/issues/547), [#547](https://github.com/dbt-labs/dbt-spark/pull/547)) ## dbt-spark 1.0.1 - April 19, 2022 @@ -16,7 +21,7 @@ ### Contributors - [@ueshin](https://github.com/ueshin) ([#285](https://github.com/dbt-labs/dbt-spark/pull/285), [#320](https://github.com/dbt-labs/dbt-spark/pull/320)) - + ## dbt-spark 1.0.0 - December 3, 2021 ### Features @@ -35,6 +40,8 @@ - [@grindheim](https://github.com/grindheim) ([#262](https://github.com/dbt-labs/dbt-spark/pull/262/)) - [@vingov](https://github.com/vingov) ([#210](https://github.com/dbt-labs/dbt-spark/pull/210)) + + ## Previous Releases For information on prior releases of dbt-spark prior to 1.0.0 please see - [0.21](https://github.com/dbt-labs/dbt-spark/blob/0.21.latest/CHANGELOG.md) diff --git a/dbt/adapters/spark/__version__.py b/dbt/adapters/spark/__version__.py index 86774adcc..409865308 100644 --- a/dbt/adapters/spark/__version__.py +++ b/dbt/adapters/spark/__version__.py @@ -1 +1 @@ -version = "1.0.1" +version = "1.0.2" diff --git a/dev_requirements.txt b/dev-requirements.txt similarity index 100% rename from dev_requirements.txt rename to dev-requirements.txt diff --git a/setup.py b/setup.py index bf1116b8b..83af357e3 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def _get_dbt_core_version(): # TODO remove old logic and add to versionBump script package_name = "dbt-spark" -package_version = "1.0.1" +package_version = "1.0.2" dbt_core_version = _get_dbt_core_version() description = """The Apache Spark adapter plugin for dbt""" diff --git a/tox.ini b/tox.ini index dbaa11e9c..0eec51060 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ passenv = DBT_* PYTEST_ADDOPTS deps = - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt [testenv:unit] allowlist_externals = @@ -23,7 +23,7 @@ passenv = PYTEST_ADDOPTS deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt [testenv:integration-spark-databricks-http] allowlist_externals = @@ -35,7 +35,7 @@ passenv = PYTEST_ADDOPTS deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e. [testenv:integration-spark-databricks-odbc-cluster] @@ -49,7 +49,7 @@ passenv = PYTEST_ADDOPTS ODBC_DRIVER deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e. [testenv:integration-spark-databricks-odbc-sql-endpoint] @@ -63,7 +63,7 @@ passenv = PYTEST_ADDOPTS ODBC_DRIVER deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e. @@ -78,5 +78,5 @@ passenv = PYTEST_ADDOPTS deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e.