diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 77a3f463f..40074e83c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.0a1 +current_version = 1.9.0b1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.9.0-b1.md b/.changes/1.9.0-b1.md new file mode 100644 index 000000000..b6daee41e --- /dev/null +++ b/.changes/1.9.0-b1.md @@ -0,0 +1,31 @@ +## dbt-spark 1.9.0-b1 - October 01, 2024 + +### Features + +- Add tests for cross-database `cast` macro ([#1028](https://github.com/dbt-labs/dbt-spark/issues/1028)) +- Cross-database `date` macro ([#1031](https://github.com/dbt-labs/dbt-spark/issues/1031)) +- Allow configuring snapshot column names ([#1096](https://github.com/dbt-labs/dbt-spark/issues/1096)) +- Support custom ODBC connection parameters via `connection_string_suffix` config ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) +- Add Microbatch Strategy to dbt-spark ([#1109](https://github.com/dbt-labs/dbt-spark/issues/1109)) + +### Fixes + +- Fix incremental python models error where Databricks could not find the temp table transaction logs ([#1033](https://github.com/dbt-labs/dbt-spark/issues/1033)) + +### Under the Hood + +- Lazy load agate to improve performance ([#1049](https://github.com/dbt-labs/dbt-spark/issues/1049)) + +### Dependencies + +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#966](https://github.com/dbt-labs/dbt-spark/pull/966)) +- Bump actions/download-artifact from 3 to 4 ([#1010](https://github.com/dbt-labs/dbt-spark/pull/1010)) +- Bump actions/upload-artifact from 3 to 4 ([#1011](https://github.com/dbt-labs/dbt-spark/pull/1011)) +- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1012](https://github.com/dbt-labs/dbt-spark/pull/1012)) +- Update wheel requirement from ~=0.42 to ~=0.43 ([#1035](https://github.com/dbt-labs/dbt-spark/pull/1035)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1073](https://github.com/dbt-labs/dbt-spark/pull/1073)) +- Loosen pin on pre-commit from 3.7.0 to 3.7 ([#1074](https://github.com/dbt-labs/dbt-spark/pull/1074)) + +### Contributors +- [@jpoley](https://github.com/jpoley) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) +- [@nilan3](https://github.com/nilan3) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) diff --git a/.changes/unreleased/Dependencies-20231219-222211.yaml b/.changes/1.9.0/Dependencies-20231219-222211.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231219-222211.yaml rename to .changes/1.9.0/Dependencies-20231219-222211.yaml diff --git a/.changes/unreleased/Dependencies-20240419-024814.yaml b/.changes/1.9.0/Dependencies-20240419-024814.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240419-024814.yaml rename to .changes/1.9.0/Dependencies-20240419-024814.yaml diff --git a/.changes/unreleased/Dependencies-20240419-024818.yaml b/.changes/1.9.0/Dependencies-20240419-024818.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240419-024818.yaml rename to .changes/1.9.0/Dependencies-20240419-024818.yaml diff --git a/.changes/unreleased/Dependencies-20240419-024820.yaml b/.changes/1.9.0/Dependencies-20240419-024820.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240419-024820.yaml rename to .changes/1.9.0/Dependencies-20240419-024820.yaml diff --git a/.changes/unreleased/Dependencies-20240503-224157.yaml b/.changes/1.9.0/Dependencies-20240503-224157.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240503-224157.yaml rename to .changes/1.9.0/Dependencies-20240503-224157.yaml diff --git a/.changes/unreleased/Dependencies-20240718-223145.yaml b/.changes/1.9.0/Dependencies-20240718-223145.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-223145.yaml rename to .changes/1.9.0/Dependencies-20240718-223145.yaml diff --git a/.changes/unreleased/Dependencies-20240718-223238.yaml b/.changes/1.9.0/Dependencies-20240718-223238.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-223238.yaml rename to .changes/1.9.0/Dependencies-20240718-223238.yaml diff --git a/.changes/unreleased/Features-20240430-185723.yaml b/.changes/1.9.0/Features-20240430-185723.yaml similarity index 100% rename from .changes/unreleased/Features-20240430-185723.yaml rename to .changes/1.9.0/Features-20240430-185723.yaml diff --git a/.changes/unreleased/Features-20240501-151904.yaml b/.changes/1.9.0/Features-20240501-151904.yaml similarity index 100% rename from .changes/unreleased/Features-20240501-151904.yaml rename to .changes/1.9.0/Features-20240501-151904.yaml diff --git a/.changes/unreleased/Features-20240903-161003.yaml b/.changes/1.9.0/Features-20240903-161003.yaml similarity index 100% rename from .changes/unreleased/Features-20240903-161003.yaml rename to .changes/1.9.0/Features-20240903-161003.yaml diff --git a/.changes/unreleased/Features-20240910-175846.yaml b/.changes/1.9.0/Features-20240910-175846.yaml similarity index 100% rename from .changes/unreleased/Features-20240910-175846.yaml rename to .changes/1.9.0/Features-20240910-175846.yaml diff --git a/.changes/unreleased/Features-20240925-125242.yaml b/.changes/1.9.0/Features-20240925-125242.yaml similarity index 100% rename from .changes/unreleased/Features-20240925-125242.yaml rename to .changes/1.9.0/Features-20240925-125242.yaml diff --git a/.changes/unreleased/Fixes-20240513-160121.yaml b/.changes/1.9.0/Fixes-20240513-160121.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240513-160121.yaml rename to .changes/1.9.0/Fixes-20240513-160121.yaml diff --git a/.changes/unreleased/Under the Hood-20240612-195629.yaml b/.changes/1.9.0/Under the Hood-20240612-195629.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240612-195629.yaml rename to .changes/1.9.0/Under the Hood-20240612-195629.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-184157.yaml b/.changes/unreleased/Breaking Changes-20241016-184157.yaml new file mode 100644 index 000000000..0a2104e60 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-184157.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:41:57.721002-04:00 +custom: + Author: mikealfare + Issue: "1121" diff --git a/.changes/unreleased/Features-20240927-133927.yaml b/.changes/unreleased/Features-20240927-133927.yaml new file mode 100644 index 000000000..ce04ac073 --- /dev/null +++ b/.changes/unreleased/Features-20240927-133927.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Enable setting current value of dbt_valid_to +time: 2024-09-27T13:39:27.268886-04:00 +custom: + Author: gshank + Issue: "1112" diff --git a/.changes/unreleased/Under the Hood-20240911-192845.yaml b/.changes/unreleased/Under the Hood-20240911-192845.yaml new file mode 100644 index 000000000..0c878f7d8 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240911-192845.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Isolating distribution testing +time: 2024-09-11T19:28:45.653064-04:00 +custom: + Author: leahwicz + Issue: "1069" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e4fc66ccc..287e5acb7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -75,7 +75,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Check out the repository @@ -126,7 +126,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -161,7 +161,7 @@ jobs: overwrite: true test-build: - name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} + name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }} if: needs.build.outputs.is_alpha == 0 @@ -172,8 +172,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: [ubuntu-latest, macos-14, windows-latest] + python-version: ["3.9", "3.10", "3.11", "3.12"] + dist-type: ["whl", "gz"] steps: - name: Set up Python ${{ matrix.python-version }} @@ -186,6 +187,7 @@ jobs: python -m pip install --user --upgrade pip python -m pip install --upgrade wheel python -m pip --version + - uses: actions/download-artifact@v4 with: name: dist @@ -194,15 +196,10 @@ jobs: - name: Show distributions run: ls -lh dist/ - - name: Install wheel distributions - run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - name: Check wheel distributions + - name: Install ${{ matrix.dist-type }} distributions run: | - python -c "import dbt.adapters.spark" - - name: Install source distributions - run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - name: Check source distributions + find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check ${{ matrix.dist-type }} distributions run: | python -c "import dbt.adapters.spark" diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 1a5090312..702ef9aea 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -37,7 +37,7 @@ defaults: shell: "bash" env: - PYTHON_TARGET_VERSION: 3.8 + PYTHON_TARGET_VERSION: 3.9 jobs: run-unit-tests: diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml index 9937463d3..d5878ec1e 100644 --- a/.github/workflows/release-prep.yml +++ b/.github/workflows/release-prep.yml @@ -84,7 +84,7 @@ defaults: shell: bash env: - PYTHON_TARGET_VERSION: 3.8 + PYTHON_TARGET_VERSION: 3.9 NOTIFICATION_PREFIX: "[Release Preparation]" jobs: @@ -448,7 +448,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Check out the repository diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fdb195262..6697bbeb5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 diff --git a/CHANGELOG.md b/CHANGELOG.md index 36a3ea69a..7679e37bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,41 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-spark 1.9.0-b1 - October 01, 2024 + +### Features + +- Add tests for cross-database `cast` macro ([#1028](https://github.com/dbt-labs/dbt-spark/issues/1028)) +- Cross-database `date` macro ([#1031](https://github.com/dbt-labs/dbt-spark/issues/1031)) +- Allow configuring snapshot column names ([#1096](https://github.com/dbt-labs/dbt-spark/issues/1096)) +- Support custom ODBC connection parameters via `connection_string_suffix` config ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) +- Add Microbatch Strategy to dbt-spark ([#1109](https://github.com/dbt-labs/dbt-spark/issues/1109)) + +### Fixes + +- Fix incremental python models error where Databricks could not find the temp table transaction logs ([#1033](https://github.com/dbt-labs/dbt-spark/issues/1033)) + +### Under the Hood + +- Lazy load agate to improve performance ([#1049](https://github.com/dbt-labs/dbt-spark/issues/1049)) + +### Dependencies + +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#966](https://github.com/dbt-labs/dbt-spark/pull/966)) +- Bump actions/download-artifact from 3 to 4 ([#1010](https://github.com/dbt-labs/dbt-spark/pull/1010)) +- Bump actions/upload-artifact from 3 to 4 ([#1011](https://github.com/dbt-labs/dbt-spark/pull/1011)) +- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1012](https://github.com/dbt-labs/dbt-spark/pull/1012)) +- Update wheel requirement from ~=0.42 to ~=0.43 ([#1035](https://github.com/dbt-labs/dbt-spark/pull/1035)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1073](https://github.com/dbt-labs/dbt-spark/pull/1073)) +- Loosen pin on pre-commit from 3.7.0 to 3.7 ([#1074](https://github.com/dbt-labs/dbt-spark/pull/1074)) + +### Contributors +- [@jpoley](https://github.com/jpoley) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) +- [@nilan3](https://github.com/nilan3) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) + + ## Previous Releases For information on prior major and minor releases, see their changelogs: -- [1.8](https://github.com/dbt-labs/dbt-spark/blob/1.8.latest/CHANGELOG.md) -- [1.7](https://github.com/dbt-labs/dbt-spark/blob/1.7.latest/CHANGELOG.md) - [1.6](https://github.com/dbt-labs/dbt-spark/blob/1.6.latest/CHANGELOG.md) - [1.5](https://github.com/dbt-labs/dbt-spark/blob/1.5.latest/CHANGELOG.md) - [1.4](https://github.com/dbt-labs/dbt-spark/blob/1.4.latest/CHANGELOG.md) diff --git a/Makefile b/Makefile index ff4c0fc1b..46b9af294 100644 --- a/Makefile +++ b/Makefile @@ -17,12 +17,12 @@ lint: ## Runs flake8 and mypy code checks against staged changes. pre-commit run --all-files .PHONY: unit -unit: ## Runs unit tests with py38. +unit: ## Runs unit tests with py39. @\ python -m pytest tests/unit .PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. +test: ## Runs unit tests with py39 and code checks against staged changes. @\ python -m pytest tests/unit; \ python dagger/run_dbt_spark_tests.py --profile spark_session \ diff --git a/dagger/run_dbt_spark_tests.py b/dagger/run_dbt_spark_tests.py index 67fa56587..6c310a6f8 100644 --- a/dagger/run_dbt_spark_tests.py +++ b/dagger/run_dbt_spark_tests.py @@ -104,7 +104,7 @@ async def test_spark(test_args): platform = dagger.Platform("linux/amd64") tst_container = ( client.container(platform=platform) - .from_("python:3.8-slim") + .from_("python:3.9-slim") .with_mounted_cache("/var/cache/apt/archives", os_reqs_cache) .with_mounted_cache("/root/.cache/pip", pip_cache) # install OS deps first so any local changes don't invalidate the cache diff --git a/dbt/adapters/spark/__version__.py b/dbt/adapters/spark/__version__.py index 6698ed64c..a4077fff2 100644 --- a/dbt/adapters/spark/__version__.py +++ b/dbt/adapters/spark/__version__.py @@ -1 +1 @@ -version = "1.9.0a1" +version = "1.9.0b1" diff --git a/dbt/include/spark/macros/materializations/snapshot.sql b/dbt/include/spark/macros/materializations/snapshot.sql index 43c4750f6..b4ef6e5d4 100644 --- a/dbt/include/spark/macros/materializations/snapshot.sql +++ b/dbt/include/spark/macros/materializations/snapshot.sql @@ -24,7 +24,12 @@ {% endif %} on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% if config.get("dbt_valid_to_current") %} + and ( DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or + DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null ) + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} diff --git a/dev-requirements.txt b/dev-requirements.txt index 055cb92f7..1fbead472 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,14 +1,13 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@snapshot_dbt_valid_to_current#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-adapters.git@test_various_snapshot_configs +git+https://github.com/dbt-labs/dbt-adapters.git@test_various_snapshot_configs#subdirectory=dbt-tests-adapter # dev ipdb~=0.13.13 -pre-commit~=3.7.0;python_version>="3.9" -pre-commit~=3.5.0;python_version<"3.9" +pre-commit~=3.7.0 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 diff --git a/docker/Dockerfile b/docker/Dockerfile index ef4574ddd..81e5e28f3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # this image gets published to GHCR for production use ARG py_version=3.11.2 -FROM python:$py_version-slim-bullseye as base +FROM python:$py_version-slim-bullseye AS base RUN apt-get update \ && apt-get dist-upgrade -y \ @@ -29,7 +29,7 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir -FROM base as dbt-spark +FROM base AS dbt-spark ARG commit_ref=main ARG extras=all diff --git a/setup.py b/setup.py index 00aeba60d..406c181d5 100644 --- a/setup.py +++ b/setup.py @@ -4,9 +4,9 @@ import re # require python 3.8 or newer -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") + print("Please upgrade to Python 3.9 or higher.") sys.exit(1) # require version of setuptools that supports find_namespace_packages @@ -41,7 +41,7 @@ def _get_plugin_version_dict(): package_name = "dbt-spark" -package_version = "1.9.0a1" +package_version = "1.9.0b1" description = """The Apache Spark adapter plugin for dbt""" odbc_extras = ["pyodbc~=5.1.0"] @@ -83,11 +83,10 @@ def _get_plugin_version_dict(): "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8", + python_requires=">=3.9", ) diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 2ca1c2211..2ecce2662 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -85,7 +85,11 @@ def model(dbt, spark): @pytest.mark.skip_profile( - "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" + "apache_spark", + "spark_session", + "databricks_sql_endpoint", + "spark_http_odbc", + "databricks_http_cluster", ) class TestChangingSchemaSpark: """ diff --git a/tests/functional/adapter/test_simple_snapshot.py b/tests/functional/adapter/test_simple_snapshot.py new file mode 100644 index 000000000..660678a99 --- /dev/null +++ b/tests/functional/adapter/test_simple_snapshot.py @@ -0,0 +1,32 @@ +from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSnapshotCheck, BaseSimpleSnapshot + +from dbt.tests.adapter.simple_snapshot.test_various_configs import ( + BaseSnapshotColumnNames, + BaseSnapshotColumnNamesFromDbtProject, + BaseSnapshotInvalidColumnNames, + BaseSnapshotDbtValidToCurrent, +) + + +class TestSnapshot(BaseSimpleSnapshot): + pass + + +class TestSnapshotCheck(BaseSnapshotCheck): + pass + + +class TestSnapshotColumnNames(BaseSnapshotColumnNames): + pass + + +class TestSnapshotColumnNamesFromDbtProject(BaseSnapshotColumnNamesFromDbtProject): + pass + + +class TestSnapshotInvalidColumnNames(BaseSnapshotInvalidColumnNames): + pass + + +class TestSnapshotDbtValidToCurrent(BaseSnapshotDbtValidToCurrent): + pass