Merge branch 'main' into 1626-configurable-batch-size-and-max-wait-li…

…mit-for-targets

.github/ISSUE_TEMPLATE/bug.yml

@@ -13,7 +13,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.34.1"
+      placeholder: "0.35.0"
     validations:
       required: true
   - type: checkboxes

.github/workflows/constraints.txt

@@ -1,7 +1,7 @@
-pip==23.3.2
+pip==24.0
 poetry==1.7.1
 poetry-plugin-export==1.6.0
 poetry-dynamic-versioning==1.2.0
-pre-commit==3.6.0
+pre-commit==3.6.1
 nox==2023.4.22
 nox-poetry==1.0.3

.github/workflows/test.yml

@@ -213,7 +213,7 @@ jobs:
       run: |
         nox -r --no-install -- xml
 
-    - uses: codecov/codecov-action@v3
+    - uses: codecov/codecov-action@v4
       with:
         fail_ci_if_error: true
         token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/version_bump.yml

@@ -74,7 +74,7 @@ jobs:
         sudo chown -R $USER:$USER .git/objects
 
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v5
+      uses: peter-evans/create-pull-request@v6
       id: create-pull-request
       with:
         token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}

.pre-commit-config.yaml

@@ -42,14 +42,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.27.3
+  rev: 0.28.0
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.14
+  rev: v0.2.1
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

CHANGELOG.md

@@ -5,6 +5,45 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.35.0 (2024-02-02)
+
+### ✨ New
+
+- [#2208](https://github.com/meltano/sdk/issues/2208) Allow users to disable schema validation in targets
+- [#2170](https://github.com/meltano/sdk/issues/2170) Generate fake data with stream maps -- _**Thanks @ReubenFrankel!**_
+- [#937](https://github.com/meltano/sdk/issues/937) Support validating configuration for any tap with a dynamic catalog
+- [#2144](https://github.com/meltano/sdk/issues/2144) Support fanning out parent record into multiple child contexts/syncs
+- [#1918](https://github.com/meltano/sdk/issues/1918) End RESTStream pagination if an empty page is received
+
+### 🐛 Fixes
+
+- [#2203](https://github.com/meltano/sdk/issues/2203) Fix serialization of arbitrary objects (e.g. `ObjectId` from mongoDB) during flattening -- _**Thanks @dgawlowsky!**_
+- [#2200](https://github.com/meltano/sdk/issues/2200) Quote column names in INSERT statement
+- [#2195](https://github.com/meltano/sdk/issues/2195) Include empty `schemas` directory in REST tap cookiecutter
+- [#2187](https://github.com/meltano/sdk/issues/2187) Replace use of deprecated `jsonschema._RefResolver` with recommended `referencing` library
+- [#2184](https://github.com/meltano/sdk/issues/2184) Reduce amount of unnecessary whitespace in Singer output
+- [#2183](https://github.com/meltano/sdk/issues/2183) Ensure `.vscode` directory is included when requested in cookiecutters and avoid failing if it does not exist
+- [#2180](https://github.com/meltano/sdk/issues/2180) Limit supported Python versions in `--about` output to existing ones
+- [#2108](https://github.com/meltano/sdk/issues/2108) Log sink name when an unhandled error occurs during setup
+- [#2158](https://github.com/meltano/sdk/issues/2158) Fix pytest plugin declaration so it can be used without requiring defining `pytest_plugins` in `conftest.py`
+- [#2105](https://github.com/meltano/sdk/issues/2105) Default handling of `ACTIVATE_VERSION` messages to soft deletes and add new `SQLConnector.delete_old_versions` method
+
+### ⚙️ Under the Hood
+
+- [#2189](https://github.com/meltano/sdk/issues/2189) Use `functools.lru_cache` instead of the stale `memoization` library (#1981)
+- [#2188](https://github.com/meltano/sdk/issues/2188) Remove unused `logger` parameter from private catalog helper functions
+- [#2143](https://github.com/meltano/sdk/issues/2143) Drop support for Python 3.7
+- [#2157](https://github.com/meltano/sdk/issues/2157) Remove `pytz` dependency and use `datetime.timezone.utc` instead of `pytz.UTC` where possible
+- [#2136](https://github.com/meltano/sdk/issues/2136) Create interface for schema validation in sinks, and implement it for `python-jsonschema` -- _**Thanks @BuzzCutNorman!**_
+- [#2130](https://github.com/meltano/sdk/issues/2130) Allow loading stream schemas from `importlib.resources.abc.Traversable` types
+
+### 📚 Documentation Improvements
+
+- [#2204](https://github.com/meltano/sdk/issues/2204) Document supported package extras
+- [#2186](https://github.com/meltano/sdk/issues/2186) Call out minimum recommended `cookiecutter` version
+- [#2168](https://github.com/meltano/sdk/issues/2168) Explain `Progress is not resumable if interrupted` in docs FAQ
+- [#2140](https://github.com/meltano/sdk/issues/2140) Update auth caching example to use `functools.cached_property`
+
 ## v0.34.1 (2023-12-19)
 
 ### 🐛 Fixes

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml

@@ -8,6 +8,7 @@ on: [push]
 jobs:
   pytest:
     runs-on: ubuntu-latest
+    continue-on-error: true
     env:
       GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }}
     strategy:
@@ -24,6 +25,7 @@ jobs:
         pip install poetry
     - name: Install dependencies
       run: |
+        poetry env use {{ '${{ matrix.python-version }}' }}
         poetry install
     - name: Test with pytest
       run: |

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml

@@ -31,12 +31,12 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-singer-sdk = { version="~=0.34.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.34.1", extras = ["testing"] }
+singer-sdk = { version="~=0.35.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml

@@ -8,6 +8,7 @@ on: [push]
 jobs:
   pytest:
     runs-on: ubuntu-latest
+    continue-on-error: true
     env:
       GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }}
     strategy:
@@ -24,6 +25,7 @@ jobs:
         pip install poetry
     - name: Install dependencies
       run: |
+        poetry env use {{ '${{ matrix.python-version }}' }}
         poetry install
     - name: Test with pytest
       run: |

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -31,15 +31,15 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.8"
 importlib-resources = { version = "==6.1.*", python = "<3.9" }
-singer-sdk = { version="~=0.34.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
 requests = "~=2.31.0"
 {%- endif %}
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.34.1", extras = ["testing"] }
+singer-sdk = { version="~=0.35.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml

@@ -8,6 +8,7 @@ on: [push]
 jobs:
   pytest:
     runs-on: ubuntu-latest
+    continue-on-error: true
     env:
       GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }}
     strategy:
@@ -24,6 +25,7 @@ jobs:
         pip install poetry
     - name: Install dependencies
       run: |
+        poetry env use {{ '${{ matrix.python-version }}' }}
         poetry install
     - name: Test with pytest
       run: |

cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml

@@ -30,15 +30,15 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-singer-sdk = { version="~=0.34.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+singer-sdk = { version="~=0.35.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.serialization_method != "SQL" %}
 requests = "~=2.31.0"
 {%- endif %}
 
 [tool.poetry.dev-dependencies]
 pytest = ">=7.4.0"
-singer-sdk = { version="~=0.34.1", extras = ["testing"] }
+singer-sdk = { version="~=0.35.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

docs/classes/singer_sdk.authenticators.APIAuthenticatorBase.rst

@@ -5,4 +5,4 @@
 
 .. autoclass:: APIAuthenticatorBase
     :members:
-    :special-members: __init__, __call__
+    :special-members: __init__, __call__

docs/conf.py

@@ -26,7 +26,7 @@
 author = "Meltano Core Team and Contributors"
 
 # The full version, including alpha/beta/rc tags
-release = "0.34.1"
+release = "0.35.0"
 
 
 # -- General configuration ---------------------------------------------------
@@ -56,7 +56,7 @@
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # Show typehints in the description, along with parameter descriptions
-autodoc_typehints = "signature"
+autodoc_typehints = "description"
 autodoc_class_signature = "separated"
 autodoc_member_order = "groupwise"
 

docs/testing.md

@@ -10,7 +10,7 @@ The Meltano SDK test framework consists of 4 main components:
 1. A runner class (`TapTestRunner` and `TargetTestRunner`), responsible for executing Taps/Targets and capturing their output.
 1. A suite dataclass, containing a list of tests.
 1. A test template classes (`TapTestTemplate`, `StreamTestTemplate`, `AttributeTestTemplate` and `TargetTestTemplate`), with methods to `.setup()`, `.test()`, `.validate()` and `.teardown()` (called in that order using `.run()`).
-1. `get_tap_test_class` and `get_target_test_class` factory methods. These wrap a `get_test_class` factory method, which takes a runner and a list of suites and return a `pytest` test class.
+1. {func}`get_tap_test_class <singer_sdk.testing.get_tap_test_class>` and {func}`get_target_test_class <singer_sdk.testing.get_target_test_class>` factory methods. These wrap a `get_test_class` factory method, which takes a runner and a list of suites and return a `pytest` test class.
 
 ## Example Usage
 
@@ -76,7 +76,7 @@ class TestTargetExample(StandardTargetTests):
 
 ## Configuring Tests
 
-Test suite behaviors can be configured by passing a `SuiteConfig` instance to the `get_test_class` functions:
+Test suite behaviors can be configured by passing a {func}`SuiteConfig <singer_sdk.testing.SuiteConfig>` instance to the `get_test_class` functions:
 
 ```python
 from singer_sdk.testing import SuiteConfig, get_tap_test_class
@@ -101,7 +101,7 @@ TestTapStackExchange = get_tap_test_class(
 )
 ```
 
-Check out [`singer_sdk/testing/config.py`](https://github.com/meltano/sdk/tree/main/singer_sdk/testing/config.py) for available config options.
+Check out [the reference](#reference) for more information on the available configuration options.
 
 ## Writing New Tests
 
@@ -127,6 +127,21 @@ my_custom_tap_tests = TestSuite(
 )
 ```
 
-This suite can now be passed to `get_tap_test_class` or `get_target_test_class` in a list of `custom_suites` along with any other suites, to generate your custom test class.
+This suite can now be passed to {func}`get_tap_test_class <singer_sdk.testing.get_tap_test_class>` or {func}`get_target_test_class <singer_sdk.testing.get_target_test_class>` in a list of `custom_suites` along with any other suites, to generate your custom test class.
 
 If your new test covers a common or general case, consider contributing to the standard test library via a pull request to [meltano/sdk](https://github.com/meltano/sdk).
+
+## Reference
+
+```{eval-rst}
+.. autofunction:: singer_sdk.testing.get_tap_test_class
+```
+
+```{eval-rst}
+.. autofunction:: singer_sdk.testing.get_target_test_class
+```
+
+```{eval-rst}
+.. autoclass:: singer_sdk.testing.SuiteConfig
+    :members:
+```

noxfile.py

@@ -42,6 +42,7 @@
     "coverage[toml]",
     "duckdb",
     "duckdb-engine",
+    "fastjsonschema",
     "pyarrow",
     "pytest",
     "pytest-benchmark",
@@ -54,13 +55,6 @@
 ]
 
 
-def _clean_py312_deps(session: Session, dependencies: list[str]) -> None:
-    """Clean dependencies for Python 3.12."""
-    if session.python == "3.12":
-        dependencies.remove("duckdb")
-        dependencies.remove("duckdb-engine")
-
-
 @session(python=main_python_version)
 def mypy(session: Session) -> None:
     """Check types with mypy."""
@@ -86,7 +80,6 @@ def mypy(session: Session) -> None:
 @session(python=python_versions)
 def tests(session: Session) -> None:
     """Execute pytest tests and compute coverage."""
-    _clean_py312_deps(session, test_dependencies)
     session.install(".[faker,parquet,s3]")
     session.install(*test_dependencies)
 
@@ -120,7 +113,6 @@ def tests(session: Session) -> None:
 @session(python=main_python_version)
 def benches(session: Session) -> None:
     """Run benchmarks."""
-    _clean_py312_deps(session, test_dependencies)
     session.install(".[s3]")
     session.install(*test_dependencies)
     sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION")
@@ -143,7 +135,6 @@ def update_snapshots(session: Session) -> None:
     """Update pytest snapshots."""
     args = session.posargs or ["-m", "snapshot"]
 
-    _clean_py312_deps(session, test_dependencies)
     session.install(".[faker]")
     session.install(*test_dependencies)
     session.run("pytest", "--snapshot-update", *args)