Merge branch 'main' into edgarrmondragon/refactor/jsonschema-draft-20…

…2012

.github/ISSUE_TEMPLATE/bug.yml

@@ -1,7 +1,9 @@
 name: Bug Report
 description: File a bug report
 title: "bug: <title>"
-labels: ["kind/Bug", "valuestream/SDK"]
+type: Bug
+assignees:
+  - edgarrmondragon
 
 body:
   - type: markdown
@@ -13,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.40.0"
+      placeholder: "0.41.0"
     validations:
       required: true
   - type: checkboxes
@@ -29,13 +31,12 @@ body:
       label: Python Version
       description: Version of Python you are using
       options:
+        - "NA"
         - "3.12"
         - "3.11"
         - "3.10"
         - "3.9"
-        - "3.8"
-        - "3.7 or earlier"
-        - "NA"
+        - "3.8 or earlier"
     validations:
       required: true
   - type: dropdown
@@ -60,7 +61,7 @@ body:
       description: What operating system you are using
       placeholder: "Windows"
     validations:
-      required: true
+      required: false
   - type: textarea
     id: what-happened
     attributes:

.github/ISSUE_TEMPLATE/docs.yml

@@ -2,6 +2,8 @@ name: Documentation change
 description: Request a documentation change
 title: "docs: <title>"
 labels: ["Documentation", "valuestream/SDK"]
+assignees:
+  - edgarrmondragon
 
 body:
   - type: markdown

.github/ISSUE_TEMPLATE/feature.yml

@@ -2,8 +2,9 @@ name: Feature request
 description: Request a new feature
 title: "feat: <title>"
 labels: ["kind/Feature", "valuestream/SDK"]
+type: Enhancement
 assignees:
-  - meltano/engineering
+  - edgarrmondragon
 
 body:
   - type: markdown
@@ -16,7 +17,8 @@ body:
       label: Feature scope
       description: Functionality this new feature would impact
       options:
-        - Taps (catalog, state, stream maps, tests, etc.)
+        - Taps (catalog, state, tests, etc.)
+        - Inline mapping (stream maps, flattening, etc.)
         - Targets (data type handling, batching, SQL object generation, tests, etc.)
         - Configuration (settings parsing, validation, etc.)
         - CLI (options, error messages, logging, etc.)

.github/workflows/api-changes.yml

@@ -38,7 +38,6 @@ jobs:
       run: |
         python -Im pip install -U pip
         pipx install griffe nox
-        pipx inject nox nox-poetry
         pipx list
 
     - name: Set REF

.github/workflows/constraints.txt

@@ -1,8 +1,5 @@
-griffe==1.3.1
+griffe==1.3.2
 pip==24.2
 poetry==1.8.3
-poetry-plugin-export==1.8.0
-poetry-dynamic-versioning==1.4.1
-pre-commit==3.8.0
+pre-commit==4.0.0
 nox==2024.4.15
-nox-poetry==1.0.3

.github/workflows/cookiecutter-e2e.yml

@@ -46,9 +46,7 @@ jobs:
         PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
       run: |
         pipx install poetry
-        pipx inject poetry poetry-plugin-export
         poetry --version
-        poetry self show plugins
 
     - uses: actions/setup-python@v5
       with:
@@ -67,7 +65,6 @@ jobs:
         PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
       run: |
         pipx install nox
-        pipx inject nox nox-poetry
         nox --version
 
     - name: Run Nox

.github/workflows/release.yml

@@ -65,7 +65,7 @@ jobs:
         name: Packages
         path: dist
     - name: Publish
-      uses: pypa/gh-action-pypi-publish@v1.10.2
+      uses: pypa/gh-action-pypi-publish@v1.10.3
 
   upload-to-release:
     name: Upload files to release

.github/workflows/test.yml

@@ -61,16 +61,6 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Install Poetry
-      env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
-      run: |
-        pipx install poetry
-        pipx inject poetry poetry-plugin-export
-        pipx inject poetry poetry-dynamic-versioning[plugin]
-        poetry --version
-        poetry self show plugins
-
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
@@ -87,7 +77,6 @@ jobs:
         PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
       run: |
         pipx install 'nox[uv]'
-        pipx inject nox nox-poetry
         nox --version
 
     - uses: actions/cache@v4
@@ -126,16 +115,6 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Install Poetry
-      env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
-      run: |
-        pipx install poetry
-        pipx inject poetry poetry-plugin-export
-        pipx inject poetry poetry-dynamic-versioning[plugin]
-        poetry --version
-        poetry self show plugins
-
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ env.NOXPYTHON }}
@@ -152,7 +131,6 @@ jobs:
         PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
       run: |
         pipx install 'nox[uv]'
-        pipx inject nox nox-poetry
         nox --version
 
     - name: Run Nox
@@ -167,16 +145,6 @@ jobs:
       NOXSESSION: coverage
     steps:
     - uses: actions/checkout@v4
-
-    - name: Install Poetry
-      env:
-        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
-      run: |
-        pipx install poetry
-        pipx inject poetry poetry-plugin-export
-        poetry --version
-        poetry self show plugins
-
     - uses: actions/setup-python@v5
       with:
         python-version: '3.12'
@@ -198,7 +166,6 @@ jobs:
         PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
       run: |
         pipx install 'nox[uv]'
-        pipx inject nox nox-poetry
         nox --version
 
     - run: nox --install-only

.github/workflows/version_bump.yml

@@ -6,6 +6,7 @@ on:
       dry_run:
         description: "Run the action without creating a PR or release draft"
         required: true
+        default: false
         type: boolean
       bump:
         description: "Version bump type"

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ ci:
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
   - id: check-json
     exclude: |
@@ -35,12 +35,6 @@ repos:
         tests/snapshots/.*
       )$
   - id: trailing-whitespace
-    exclude: |
-        (?x)^(
-            .bumpversion.cfg|
-            singer_sdk/helpers/_simpleeval.py|
-            tests/core/test_simpleeval.py
-        )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
   rev: 0.29.3
@@ -50,7 +44,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.8
+  rev: v0.6.9
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

.readthedocs.yml

@@ -1,7 +1,7 @@
 version: 2
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
     python: "3.12"
 

CHANGELOG.md

@@ -5,6 +5,41 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.41.0 (2024-10-02)
+
+### ✨ New
+
+- [#2667](https://github.com/meltano/sdk/issues/2667) Support stream aliasing of `BATCH` messages via stream maps -- _**Thanks @ReubenFrankel!**_
+- [#2651](https://github.com/meltano/sdk/issues/2651) SQL taps now emit schemas with `maxLength` when applicable
+- [#2618](https://github.com/meltano/sdk/issues/2618) Developers can now more easily override the mapping from SQL column type to JSON schema
+
+### 🐛 Fixes
+
+- [#2697](https://github.com/meltano/sdk/issues/2697) All HTTP timeout exceptions are now retried in REST and GraphQL streams
+- [#2683](https://github.com/meltano/sdk/issues/2683) A clear error message is now emitted when flattening is enabled but `flattening_max_depth` is not set
+- [#2665](https://github.com/meltano/sdk/issues/2665) Mapped datetime values are now typed as `date-time` strings in the schema message -- _**Thanks @gregkoutsimp!**_
+- [#2663](https://github.com/meltano/sdk/issues/2663) Properties dropped using `None` or `__NULL__` in stream maps are now also removed from the schema `required` array
+
+### ⚙️ Under the Hood
+
+- [#2696](https://github.com/meltano/sdk/issues/2696) Use tox without installing Poetry explicitly in workflows
+- [#2654](https://github.com/meltano/sdk/issues/2654) Added a generic `FileStream` (still in active development!)
+- [#2695](https://github.com/meltano/sdk/issues/2695) Update dependencies in templates
+- [#2661](https://github.com/meltano/sdk/issues/2661) Drop support for Python 3.8 in templates
+- [#2670](https://github.com/meltano/sdk/issues/2670) Deprecated `Faker` class in stream maps
+- [#2666](https://github.com/meltano/sdk/issues/2666) Remove non-functional `record-flattening` capability -- _**Thanks @ReubenFrankel!**_
+- [#2652](https://github.com/meltano/sdk/issues/2652) Renamed `SQLConnector.type_mapping` to `SQLConnector.sql_to_jsonschema`
+- [#2647](https://github.com/meltano/sdk/issues/2647) Use future `warnings.deprecated` decorator
+
+### 📚 Documentation Improvements
+
+- [#2658](https://github.com/meltano/sdk/issues/2658) Added more versions when stuff changed or was added
+
+### 📦 Packaging changes
+
+- [#2694](https://github.com/meltano/sdk/issues/2694) Removed unused backport `importlib_resources` dependency in tap template
+- [#2664](https://github.com/meltano/sdk/issues/2664) Added a constraint on setuptools <= 70.3.0 to fix incompatibility with some dependencies
+
 ## v0.40.0 (2024-09-02)
 
 ### ✨ New

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml

@@ -30,12 +30,12 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.9"
-singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+singer-sdk = { version="~=0.41.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8"
-singer-sdk = { version="~=0.40.0", extras = ["testing"] }
+singer-sdk = { version="~=0.41.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py

@@ -23,6 +23,7 @@ class {{ cookiecutter.name }}Mapper(InlineMapper):
         th.Property(
             "example_config",
             th.StringType,
+            title="Example Configuration",
             description="An example config, replace or remove based on your needs.",
         ),
     ).to_dict()

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -29,7 +29,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.9"
-singer-sdk = { version="~=0.40.0", extras = [
+singer-sdk = { version="~=0.41.0", extras = [
     {%- if cookiecutter.auth_method == "JWT" -%}"jwt", {% endif -%}
     {%- if cookiecutter.faker_extra -%}"faker",{%- endif -%}
 ] }
@@ -41,9 +41,9 @@ requests = "~=2.32.3"
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8"
 {%- if cookiecutter.auth_method == "JWT" %}
-singer-sdk = { version="~=0.40.0", extras = ["jwt", "testing"] }
+singer-sdk = { version="~=0.41.0", extras = ["jwt", "testing"] }
 {%- else %}
-singer-sdk = { version="~=0.40.0", extras = ["testing"] }
+singer-sdk = { version="~=0.41.0", extras = ["testing"] }
 {%- endif %}
 
 [tool.poetry.extras]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py

@@ -31,12 +31,14 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
             th.StringType,
             required=True,
             secret=True,  # Flag config as protected.
+            title="Auth Token",
             description="The token to authenticate against the API service",
         ),
         th.Property(
             "project_ids",
             th.ArrayType(th.StringType),
             required=True,
+            title="Project IDs",
             description="Project IDs to replicate",
         ),
         th.Property(
@@ -47,6 +49,7 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
         th.Property(
             "api_url",
             th.StringType,
+            title="API URL",
             default="https://api.mysample.com",
             description="The url for the API service",
         ),

cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml

@@ -29,15 +29,15 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.9"
-singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+singer-sdk = { version="~=0.41.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.serialization_method != "SQL" %}
 requests = "~=2.32.3"
 {%- endif %}
 
 [tool.poetry.dev-dependencies]
 pytest = ">=8"
-singer-sdk = { version="~=0.40.0", extras = ["testing"] }
+singer-sdk = { version="~=0.41.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/target.py

@@ -23,23 +23,27 @@ class Target{{ cookiecutter.destination_name }}({{ target_class }}):
             "sqlalchemy_url",
             th.StringType,
             secret=True,  # Flag config as protected.
+            title="SQLAlchemy URL",
             description="SQLAlchemy connection string",
         ),
         {%- else %}
         th.Property(
             "filepath",
             th.StringType,
+            title="Output File Path",
             description="The path to the target output file",
         ),
         th.Property(
             "file_naming_scheme",
             th.StringType,
+            title="File Naming Scheme",
             description="The scheme with which output files will be named",
         ),
         th.Property(
             "auth_token",
             th.StringType,
             secret=True,  # Flag config as protected.
+            title="Auth Token",
             description="The path to the target output file",
         ),
         {%- endif %}