From f0a01100abb304582856320f099db3b1b07cd44c Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 23 Dec 2022 12:58:30 -0800 Subject: [PATCH] 1.1 update version-bump GHA (#562) * update version-bump GHA * align dev-requirements file naming w core * fix dev-requirements name * Bumping version to 1.1.1rc0 and generate CHANGELOG (#568) Co-authored-by: Github Build Bot * Bumping version to 1.1.1 and generate CHANGELOG (#571) Co-authored-by: Github Build Bot * skip databricks integ tests * fix dev-requirements naming * fix circle CI config Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Github Build Bot --- .bumpversion.cfg | 2 +- .changes/1.1.1.md | 11 ++ .../unreleased/Features-20220810-131800.yaml | 7 -- .../unreleased/Fixes-20221116-234601.yaml | 7 -- .../Under the Hood-20221202-140724.yaml | 7 -- .../Under the Hood-20221209-143611.yaml | 7 -- .circleci/config.yml | 42 ++++--- .github/workflows/main.yml | 2 +- .github/workflows/release.yml | 47 ++++---- .github/workflows/version-bump.yml | 103 +++--------------- CHANGELOG.md | 13 ++- dbt/adapters/spark/__version__.py | 2 +- dev_requirements.txt => dev-requirements.txt | 0 setup.py | 2 +- tox.ini | 14 +-- 15 files changed, 94 insertions(+), 172 deletions(-) create mode 100644 .changes/1.1.1.md delete mode 100644 .changes/unreleased/Features-20220810-131800.yaml delete mode 100644 .changes/unreleased/Fixes-20221116-234601.yaml delete mode 100644 .changes/unreleased/Under the Hood-20221202-140724.yaml delete mode 100644 .changes/unreleased/Under the Hood-20221209-143611.yaml rename dev_requirements.txt => dev-requirements.txt (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 88442026c..30e7913f7 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.1.0 +current_version = 1.1.1 parse = (?P\d+) \.(?P\d+) \.(?P\d+) diff --git a/.changes/1.1.1.md b/.changes/1.1.1.md new file mode 100644 index 000000000..a2933212d --- /dev/null +++ b/.changes/1.1.1.md @@ -0,0 +1,11 @@ +## dbt-spark 1.1.1 - December 20, 2022 +### Features +- backport changie to 1.1.latest ([#417](https://github.com/dbt-labs/dbt-spark/issues/417), [#425](https://github.com/dbt-labs/dbt-spark/pull/425)) +### Fixes +- Support new error messages in the future Spark. ([#515](https://github.com/dbt-labs/dbt-spark/issues/515), [#520](https://github.com/dbt-labs/dbt-spark/pull/520)) +### Under the Hood +- updating python version in tox ([#536](https://github.com/dbt-labs/dbt-spark/issues/536), [#534](https://github.com/dbt-labs/dbt-spark/pull/534)) +- fix post new release tox issues around passenv and allowlist_externals ([#546](https://github.com/dbt-labs/dbt-spark/issues/546), [#546](https://github.com/dbt-labs/dbt-spark/pull/546)) + +### Contributors +- [@ueshin](https://github.com/ueshin) ([#520](https://github.com/dbt-labs/dbt-spark/pull/520)) diff --git a/.changes/unreleased/Features-20220810-131800.yaml b/.changes/unreleased/Features-20220810-131800.yaml deleted file mode 100644 index dbb56ed02..000000000 --- a/.changes/unreleased/Features-20220810-131800.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Features -body: backport changie to 1.1.latest -time: 2022-08-10T13:18:00.106195-05:00 -custom: - Author: mcknight-42 - Issue: "417" - PR: "425" diff --git a/.changes/unreleased/Fixes-20221116-234601.yaml b/.changes/unreleased/Fixes-20221116-234601.yaml deleted file mode 100644 index 3e394c87b..000000000 --- a/.changes/unreleased/Fixes-20221116-234601.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: Support new error messages in the future Spark. -time: 2022-11-16T23:46:01.899921861Z -custom: - Author: ueshin - Issue: "515" - PR: "520" diff --git a/.changes/unreleased/Under the Hood-20221202-140724.yaml b/.changes/unreleased/Under the Hood-20221202-140724.yaml deleted file mode 100644 index 88fee7bea..000000000 --- a/.changes/unreleased/Under the Hood-20221202-140724.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Under the Hood -body: updating python version in tox -time: 2022-12-02T14:07:24.377668-06:00 -custom: - Author: McKnight-42 - Issue: "536" - PR: "534" diff --git a/.changes/unreleased/Under the Hood-20221209-143611.yaml b/.changes/unreleased/Under the Hood-20221209-143611.yaml deleted file mode 100644 index b0f34875f..000000000 --- a/.changes/unreleased/Under the Hood-20221209-143611.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Under the Hood -body: fix post new release tox issues around passenv and allowlist_externals -time: 2022-12-09T14:36:11.005107-06:00 -custom: - Author: Mcknight-42 - Issue: "546" - PR: "546" diff --git a/.circleci/config.yml b/.circleci/config.yml index 34e449acf..53c1849d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,21 +80,27 @@ jobs: environment: DBT_INVOCATION_ENV: circle DBT_DATABRICKS_RETRY_ALL: True + DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" + DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" + DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com" docker: - image: fishtownanalytics/test-container:10 steps: - checkout - - run: - name: Run integration tests - command: tox -e integration-spark-databricks-http - no_output_timeout: 1h - - store_artifacts: - path: ./logs + # - run: + # name: Run integration tests + # command: tox -e integration-spark-databricks-http + # no_output_timeout: 1h + # - store_artifacts: + # path: ./logs integration-spark-databricks-odbc-cluster: &databricks-odbc environment: DBT_INVOCATION_ENV: circle ODBC_DRIVER: Simba # TODO: move env var to Docker image + DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" + DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" + DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com" docker: # image based on `fishtownanalytics/test-container` w/ Simba ODBC Spark driver installed - image: 828731156495.dkr.ecr.us-east-1.amazonaws.com/dbt-spark-odbc-test-container:latest @@ -103,23 +109,23 @@ jobs: aws_secret_access_key: $AWS_SECRET_ACCESS_KEY_STAGING steps: - checkout - - run: - name: Run integration tests - command: tox -e integration-spark-databricks-odbc-cluster - no_output_timeout: 1h - - store_artifacts: - path: ./logs + # - run: + # name: Run integration tests + # command: tox -e integration-spark-databricks-odbc-cluster + # no_output_timeout: 1h + # - store_artifacts: + # path: ./logs integration-spark-databricks-odbc-endpoint: <<: *databricks-odbc steps: - checkout - - run: - name: Run integration tests - command: tox -e integration-spark-databricks-odbc-sql-endpoint - no_output_timeout: 1h - - store_artifacts: - path: ./logs + # - run: + # name: Run integration tests + # command: tox -e integration-spark-databricks-odbc-sql-endpoint + # no_output_timeout: 1h + # - store_artifacts: + # path: ./logs workflows: version: 2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1fd714fac..f8ec16627 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,7 @@ jobs: python -m pip install mypy==0.942 mypy --version python -m pip install -r requirements.txt - python -m pip install -r dev_requirements.txt + python -m pip install -r dev-requirements.txt dbt --version unit: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b25ea884e..30544b277 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,28 +3,28 @@ name: Build and Release on: workflow_dispatch: - + # Release version number that must be updated for each release env: - version_number: '0.20.0rc2' + version_number: "0.20.0rc2" -jobs: +jobs: Test: runs-on: ubuntu-latest steps: - name: Setup Python uses: actions/setup-python@v2.2.2 - with: - python-version: '3.8' - + with: + python-version: "3.8" + - uses: actions/checkout@v2 - - name: Test release + - name: Test release run: | python3 -m venv env source env/bin/activate sudo apt-get install libsasl2-dev - pip install -r dev_requirements.txt + pip install -r dev-requirements.txt pip install twine wheel setuptools python setup.py sdist bdist_wheel pip install dist/dbt-spark-*.tar.gz @@ -38,9 +38,9 @@ jobs: steps: - name: Setup Python uses: actions/setup-python@v2.2.2 - with: - python-version: '3.8' - + with: + python-version: "3.8" + - uses: actions/checkout@v2 - name: Bumping version @@ -48,7 +48,7 @@ jobs: python3 -m venv env source env/bin/activate sudo apt-get install libsasl2-dev - pip install -r dev_requirements.txt + pip install -r dev-requirements.txt bumpversion --config-file .bumpversion-dbt.cfg patch --new-version ${{env.version_number}} bumpversion --config-file .bumpversion.cfg patch --new-version ${{env.version_number}} --allow-dirty git status @@ -56,11 +56,11 @@ jobs: - name: Commit version bump and tag uses: EndBug/add-and-commit@v7 with: - author_name: 'Leah Antkiewicz' - author_email: 'leah.antkiewicz@dbtlabs.com' - message: 'Bumping version to ${{env.version_number}}' + author_name: "Leah Antkiewicz" + author_email: "leah.antkiewicz@dbtlabs.com" + message: "Bumping version to ${{env.version_number}}" tag: v${{env.version_number}} - + # Need to set an output variable because env variables can't be taken as input # This is needed for the next step with releasing to GitHub - name: Find release type @@ -69,7 +69,7 @@ jobs: IS_PRERELEASE: ${{ contains(env.version_number, 'rc') || contains(env.version_number, 'b') }} run: | echo ::set-output name=isPrerelease::$IS_PRERELEASE - + - name: Create GitHub release uses: actions/create-release@v1 env: @@ -88,7 +88,7 @@ jobs: # or $ pip install "dbt-spark[PyHive]==${{env.version_number}}" ``` - + PypiRelease: name: Pypi release runs-on: ubuntu-latest @@ -97,13 +97,13 @@ jobs: steps: - name: Setup Python uses: actions/setup-python@v2.2.2 - with: - python-version: '3.8' - + with: + python-version: "3.8" + - uses: actions/checkout@v2 with: ref: v${{env.version_number}} - + - name: Release to pypi env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} @@ -112,8 +112,7 @@ jobs: python3 -m venv env source env/bin/activate sudo apt-get install libsasl2-dev - pip install -r dev_requirements.txt + pip install -r dev-requirements.txt pip install twine wheel setuptools python setup.py sdist bdist_wheel twine upload --non-interactive dist/dbt_spark-${{env.version_number}}-py3-none-any.whl dist/dbt-spark-${{env.version_number}}.tar.gz - diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index 7fb8bb6eb..bde34d683 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -1,18 +1,15 @@ # **what?** -# This workflow will take a version number and a dry run flag. With that -# it will run versionbump to update the version number everywhere in the -# code base and then generate an update Docker requirements file. If this -# is a dry run, a draft PR will open with the changes. If this isn't a dry -# run, the changes will be committed to the branch this is run on. +# This workflow will take the new version number to bump to. With that +# it will run versionbump to update the version number everywhere in the +# code base and then run changie to create the corresponding changelog. +# A PR will be created with the changes that can be reviewed before committing. # **why?** -# This is to aid in releasing dbt and making sure we have updated -# the versions and Docker requirements in all places. +# This is to aid in releasing dbt and making sure we have updated +# the version in all places and generated the changelog. # **when?** -# This is triggered either manually OR -# from the repository_dispatch event "version-bump" which is sent from -# the dbt-release repo Action +# This is triggered manually name: Version Bump @@ -20,84 +17,12 @@ on: workflow_dispatch: inputs: version_number: - description: 'The version number to bump to' + description: 'The version number to bump to (ex. 1.2.0, 1.3.0b1)' required: true - is_dry_run: - description: 'Creates a draft PR to allow testing instead of committing to a branch' - required: true - default: 'true' - repository_dispatch: - types: [version-bump] - -jobs: - bump: - runs-on: ubuntu-latest - steps: - - name: Check out the repository - uses: actions/checkout@v2 - - - name: Set version and dry run values - id: variables - env: - VERSION_NUMBER: "${{ github.event.client_payload.version_number == '' && github.event.inputs.version_number || github.event.client_payload.version_number }}" - IS_DRY_RUN: "${{ github.event.client_payload.is_dry_run == '' && github.event.inputs.is_dry_run || github.event.client_payload.is_dry_run }}" - run: | - echo Repository dispatch event version: ${{ github.event.client_payload.version_number }} - echo Repository dispatch event dry run: ${{ github.event.client_payload.is_dry_run }} - echo Workflow dispatch event version: ${{ github.event.inputs.version_number }} - echo Workflow dispatch event dry run: ${{ github.event.inputs.is_dry_run }} - echo ::set-output name=VERSION_NUMBER::$VERSION_NUMBER - echo ::set-output name=IS_DRY_RUN::$IS_DRY_RUN - - - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - - name: Install python dependencies - run: | - sudo apt-get install libsasl2-dev - python3 -m venv env - source env/bin/activate - pip install --upgrade pip - - - name: Create PR branch - if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }} - run: | - git checkout -b bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID - git push origin bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID - git branch --set-upstream-to=origin/bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID - - - name: Bumping version - run: | - source env/bin/activate - pip install -r dev_requirements.txt - env/bin/bumpversion --allow-dirty --new-version ${{steps.variables.outputs.VERSION_NUMBER}} major - git status - - - name: Commit version bump directly - uses: EndBug/add-and-commit@v7 - if: ${{ steps.variables.outputs.IS_DRY_RUN == 'false' }} - with: - author_name: 'Github Build Bot' - author_email: 'buildbot@fishtownanalytics.com' - message: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}' - - - name: Commit version bump to branch - uses: EndBug/add-and-commit@v7 - if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }} - with: - author_name: 'Github Build Bot' - author_email: 'buildbot@fishtownanalytics.com' - message: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}' - branch: 'bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}' - push: 'origin origin/bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}' - - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 - if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }} - with: - author: 'Github Build Bot ' - draft: true - base: ${{github.ref}} - title: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}' - branch: 'bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}' +jobs: + version_bump_and_changie: + uses: dbt-labs/actions/.github/workflows/version-bump.yml@main + with: + version_number: ${{ inputs.version_number }} + secrets: inherit # ok since what we are calling is internally maintained diff --git a/CHANGELOG.md b/CHANGELOG.md index 12689bfc0..022ba404a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,17 @@ - Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-spark 1.1.1 - December 20, 2022 +### Features +- backport changie to 1.1.latest ([#417](https://github.com/dbt-labs/dbt-spark/issues/417), [#425](https://github.com/dbt-labs/dbt-spark/pull/425)) +### Fixes +- Support new error messages in the future Spark. ([#515](https://github.com/dbt-labs/dbt-spark/issues/515), [#520](https://github.com/dbt-labs/dbt-spark/pull/520)) +### Under the Hood +- updating python version in tox ([#536](https://github.com/dbt-labs/dbt-spark/issues/536), [#534](https://github.com/dbt-labs/dbt-spark/pull/534)) +- fix post new release tox issues around passenv and allowlist_externals ([#546](https://github.com/dbt-labs/dbt-spark/issues/546), [#546](https://github.com/dbt-labs/dbt-spark/pull/546)) +### Contributors +- [@ueshin](https://github.com/ueshin) ([#520](https://github.com/dbt-labs/dbt-spark/pull/520)) ## dbt-spark 1.1.0 - April 28, 2022 ### Features @@ -25,11 +35,10 @@ - Configure insert_overwrite models to use parquet ([#301](https://github.com/dbt-labs/dbt-spark/pull/301)) ### Contributors -- [@JCZuurmond](https://github.com/dbt-labs/dbt-spark/pull/279) ([#279](https://github.com/dbt-labs/dbt-spark/pull/279)) +- [@JCZuurmond](https://github.com/dbt-labs/dbt-spark/pull/279) ( [#279](https://github.com/dbt-labs/dbt-spark/pull/279)) - [@ueshin](https://github.com/ueshin) ([#320](https://github.com/dbt-labs/dbt-spark/pull/320)) - [@amychen1776](https://github.com/amychen1776) ([#288](https://github.com/dbt-labs/dbt-spark/pull/288)) - [@ueshin](https://github.com/ueshin) ([#285](https://github.com/dbt-labs/dbt-spark/pull/285)) - ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.0](https://github.com/dbt-labs/dbt-spark/blob/1.0.latest/CHANGELOG.md) diff --git a/dbt/adapters/spark/__version__.py b/dbt/adapters/spark/__version__.py index b2b60a550..64c8c21b1 100644 --- a/dbt/adapters/spark/__version__.py +++ b/dbt/adapters/spark/__version__.py @@ -1 +1 @@ -version = "1.1.0" +version = "1.1.1" diff --git a/dev_requirements.txt b/dev-requirements.txt similarity index 100% rename from dev_requirements.txt rename to dev-requirements.txt diff --git a/setup.py b/setup.py index 2f5278d50..6e07e45ff 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def _get_dbt_core_version(): package_name = "dbt-spark" -package_version = "1.1.0" +package_version = "1.1.1" dbt_core_version = _get_dbt_core_version() description = """The Apache Spark adapter plugin for dbt""" diff --git a/tox.ini b/tox.ini index e84c31d43..7f08b3b97 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ passenv = DBT_* PYTEST_ADDOPTS deps = - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt [testenv:{unit,py37,py38,py39,py310,py}] allowlist_externals = @@ -22,7 +22,7 @@ passenv = PYTEST_ADDOPTS deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt [testenv:integration-spark-databricks-http] allowlist_externals = @@ -34,7 +34,7 @@ passenv = PYTEST_ADDOPTS deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e. [testenv:integration-spark-databricks-odbc-cluster] @@ -49,7 +49,7 @@ passenv = ODBC_DRIVER deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e. [testenv:integration-spark-databricks-odbc-sql-endpoint] @@ -64,7 +64,7 @@ passenv = ODBC_DRIVER deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e. @@ -79,7 +79,7 @@ passenv = PYTEST_ADDOPTS deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e. [testenv:integration-spark-session] @@ -93,5 +93,5 @@ passenv = PIP_CACHE_DIR deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/dev_requirements.txt + -r{toxinidir}/dev-requirements.txt -e.[session]