Merge branch 'main' into 1882-bug-a-field-referencing-record-in-strea…

…ms-maps-gets-incorrectly-mapped-to-a-string-type-in-the-schema

.github/ISSUE_TEMPLATE/bug.yml

@@ -15,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.31.0"
+      placeholder: "0.32.0b3"
     validations:
       required: true
   - type: checkboxes

.github/workflows/codeql-analysis.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3.5.3
+      uses: actions/checkout@v4.0.0
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/constraints.txt

@@ -1,5 +1,5 @@
 pip==23.2.1
-poetry==1.5.1
-pre-commit==3.3.3
+poetry==1.6.1
+pre-commit==3.4.0
 nox==2023.4.22
 nox-poetry==1.0.3

.github/workflows/cookiecutter-e2e.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3.5.3
+      uses: actions/checkout@v4.0.0
 
     - name: Upgrade pip
       env:

.github/workflows/dependency-review.yml

@@ -16,11 +16,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repository
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4.0.0
 
       - name: GitHub dependency vulnerability check
         if: ${{ github.event_name == 'pull_request_target' }}
-        uses: actions/dependency-review-action@v3.0.6
+        uses: actions/dependency-review-action@v3.1.0
         with:
           fail-on-severity: high
 

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3.5.3
+      uses: actions/checkout@v4.0.0
 
     - name: Set up Python
       uses: actions/setup-python@v4.7.0
@@ -56,4 +56,4 @@ jobs:
         file_glob: true
 
     - name: Publish
-      uses: pypa/gh-action-pypi-publish@v1.8.8
+      uses: pypa/gh-action-pypi-publish@v1.8.10

.github/workflows/test.yml

@@ -55,7 +55,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3.5.3
+      uses: actions/checkout@v4.0.0
 
     - name: Install Poetry
       env:
@@ -115,7 +115,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3.5.3
+      uses: actions/checkout@v4.0.0
 
     - name: Install Poetry
       env:
@@ -157,7 +157,7 @@ jobs:
     needs: tests
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3.5.3
+      uses: actions/checkout@v4.0.0
 
     - name: Install Poetry
       run: |

.github/workflows/version_bump.yml

@@ -35,7 +35,7 @@ jobs:
       pull-requests: write  # to create and update PRs
 
     steps:
-    - uses: actions/checkout@v3.5.3
+    - uses: actions/checkout@v4.0.0
       with:
         fetch-depth: 0
 
@@ -47,7 +47,7 @@ jobs:
 
     - name: Bump version
       id: cz-bump
-      uses: commitizen-tools/commitizen-action@0.18.2
+      uses: commitizen-tools/commitizen-action@0.20.0
       with:
         increment: ${{ github.event.inputs.bump != 'auto' && github.event.inputs.bump || '' }}
         prerelease: ${{ github.event.inputs.prerelease != 'none' && github.event.inputs.prerelease || '' }}
@@ -66,8 +66,7 @@ jobs:
         body_path: _changelog_fragment.md
         tag_name: v${{ steps.cz-bump.outputs.version }}
         prerelease: ${{ github.event.inputs.prerelease != 'none' }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}
 
     - name: Set repo file permissions
       run: |
@@ -77,6 +76,7 @@ jobs:
       uses: peter-evans/create-pull-request@v5
       id: create-pull-request
       with:
+        token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}
         commit-message: "chore: Bump package version"
         title: "chore: Release v${{ steps.cz-bump.outputs.version }}"
         body: |

.pre-commit-config.yaml

@@ -36,14 +36,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.23.3
+  rev: 0.26.3
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.282
+  rev: v0.0.290
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
@@ -53,7 +53,7 @@ repos:
       )$
 
 - repo: https://github.com/psf/black
-  rev: 23.7.0
+  rev: 23.9.1
   hooks:
   - id: black
     exclude: |
@@ -76,6 +76,6 @@ repos:
         )$
 
 - repo: https://github.com/python-poetry/poetry
-  rev: 1.5.0
+  rev: 1.6.0
   hooks:
   - id: poetry-check

CHANGELOG.md

@@ -5,6 +5,48 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.32.0b3 (2023-09-22)
+
+### 🐛 Fixes
+
+- [#1977](https://github.com/meltano/sdk/issues/1977) Fix hanging downstream tests in tap-postgres
+- [#1970](https://github.com/meltano/sdk/issues/1970) Warn instead of crashing when schema helpers cannot append `null` to types
+
+## v0.32.0b2 (2023-09-15)
+
+### ⚡ Performance Improvements
+
+- [#1962](https://github.com/meltano/sdk/issues/1962) Ensure `raw_schema` in stream mapper is immutable
+
+## v0.32.0b1 (2023-09-13)
+
+### ✨ New
+
+- [#1893](https://github.com/meltano/sdk/issues/1893) Standard configurable load methods
+- [#1861](https://github.com/meltano/sdk/issues/1861) SQLTap connector instance shared with streams -- _**Thanks @BuzzCutNorman!**_
+
+### 🐛 Fixes
+
+- [#1954](https://github.com/meltano/sdk/issues/1954) Missing begin()s related to SQLAlchemy 2.0 -- _**Thanks @andyoneal!**_
+- [#1951](https://github.com/meltano/sdk/issues/1951) Ensure SQL streams are sorted when a replication key is set
+- [#1949](https://github.com/meltano/sdk/issues/1949) Retry SQLAlchemy engine creation for adapters without JSON SerDe support
+- [#1939](https://github.com/meltano/sdk/issues/1939) Handle `decimal.Decimal` instances in flattening
+- [#1927](https://github.com/meltano/sdk/issues/1927) Handle replication key not found in stream schema -- _**Thanks @mjsqu!**_
+
+### 📚 Documentation Improvements
+
+- [#1925](https://github.com/meltano/sdk/issues/1925) Add viztracer command for testing targets -- _**Thanks @mjsqu!**_
+
+## v0.31.1 (2023-08-17)
+
+### ✨ New
+
+- [#1905](https://github.com/meltano/sdk/issues/1905) Add email field and use human-readable questions in templates
+
+### 🐛 Fixes
+
+- [#1913](https://github.com/meltano/sdk/issues/1913) Fix tap tests for multiple test classes with different input catalogs
+
 ## v0.31.0 (2023-08-07)
 
 ### ✨ New

cookiecutter/mapper-template/cookiecutter.json

@@ -1,9 +1,19 @@
 {
   "name": "MyMapperName",
   "admin_name": "FirstName LastName",
+  "admin_email": "firstname.lastname@example.com",
   "mapper_id": "mapper-{{ cookiecutter.name.lower() }}",
   "library_name": "{{ cookiecutter.mapper_id.replace('-', '_') }}",
   "variant": "None (Skip)",
   "include_ci_files": ["GitHub", "None (Skip)"],
-  "license": ["Apache-2.0"]
+  "license": ["Apache-2.0"],
+  "__prompts__": {
+    "name": "The name of the mapper, in CamelCase",
+    "admin_name": "Provide your [bold yellow]full name[/]",
+    "admin_email": "Provide your [bold yellow]email[/]",
+    "mapper_id": "The ID of the tap, in kebab-case",
+    "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
+    "include_ci_files": "Whether to include CI files for a common CI services",
+    "license": "The license for the project"
+  }
 }

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml

@@ -7,7 +7,7 @@ name = "{{cookiecutter.mapper_id}}"
 version = "0.0.1"
 description = "`{{cookiecutter.mapper_id}}` is a Singer mapper {{cookiecutter.name}}, built with the Meltano Singer SDK."
 readme = "README.md"
-authors = ["{{ cookiecutter.admin_name }}"]
+authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"]
 keywords = [
     "ELT",
     "Mapper",
@@ -21,13 +21,13 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = "<3.12,>=3.7.1"
-singer-sdk = { version="^0.30.0" }
-fs-s3fs = { version = "^1.1.1", optional = true }
+python = ">=3.7.1,<4"
+singer-sdk = { version="~=0.32.0b3" }
+fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.4.0"
-singer-sdk = { version="^0.30.0", extras = ["testing"] }
+pytest = ">=7.4.0"
+singer-sdk = { version="~=0.32.0b3", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/tap-template/cookiecutter.json

@@ -1,6 +1,7 @@
 {
   "source_name": "MySourceName",
   "admin_name": "FirstName LastName",
+  "admin_email": "firstname.lastname@example.com",
   "tap_id": "tap-{{ cookiecutter.source_name.lower() }}",
   "library_name": "{{ cookiecutter.tap_id.replace('-', '_') }}",
   "variant": "None (Skip)",
@@ -14,5 +15,16 @@
     "Custom or N/A"
   ],
   "include_ci_files": ["GitHub", "None (Skip)"],
-  "license": ["Apache-2.0"]
+  "license": ["Apache-2.0"],
+  "__prompts__": {
+    "source_name": "The name of the source, in CamelCase",
+    "admin_name": "Provide your [bold yellow]full name[/]",
+    "admin_email": "Provide your [bold yellow]email[/]",
+    "tap_id": "The ID of the tap, in kebab-case",
+    "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
+    "stream_type": "The type of stream the source provides",
+    "auth_method": "The [bold red]authentication[/] method used by the source, for REST and GraphQL sources",
+    "include_ci_files": "Whether to include CI files for a common CI services",
+    "license": "The license for the project"
+  }
 }

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -7,7 +7,7 @@ name = "{{cookiecutter.tap_id}}"
 version = "0.0.1"
 description = "`{{cookiecutter.tap_id}}` is a Singer tap for {{cookiecutter.source_name}}, built with the Meltano Singer SDK."
 readme = "README.md"
-authors = ["{{ cookiecutter.admin_name }}"]
+authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"]
 keywords = [
     "ELT",
     "{{cookiecutter.source_name}}",
@@ -20,19 +20,19 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = "<3.12,>=3.7.1"
-singer-sdk = { version="^0.31.0" }
-fs-s3fs = { version = "^1.1.1", optional = true }
+python = ">=3.7.1,<4"
+singer-sdk = { version="~=0.32.0b3" }
+fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
-requests = "^2.31.0"
+requests = "~=2.31.0"
 {%- endif %}
 {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
-cached-property = "^1" # Remove after Python 3.7 support is dropped
+cached-property = "~=1" # Remove after Python 3.7 support is dropped
 {%- endif %}
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.4.0"
-singer-sdk = { version="^0.31.0", extras = ["testing"] }
+pytest = ">=7.4.0"
+singer-sdk = { version="~=0.32.0b3", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/target-template/cookiecutter.json

@@ -1,10 +1,21 @@
 {
   "destination_name": "MyDestinationName",
   "admin_name": "FirstName LastName",
+  "admin_email": "firstname.lastname@example.com",
   "target_id": "target-{{ cookiecutter.destination_name.lower() }}",
   "library_name": "{{ cookiecutter.target_id.replace('-', '_') }}",
   "variant": "None (Skip)",
   "serialization_method": ["Per record", "Per batch", "SQL"],
   "include_ci_files": ["GitHub", "None (Skip)"],
-  "license": ["Apache-2.0"]
+  "license": ["Apache-2.0"],
+  "__prompts__": {
+    "name": "The name of the mapper, in CamelCase",
+    "admin_name": "Provide your [bold yellow]full name[/]",
+    "admin_email": "Provide your [bold yellow]email[/]",
+    "mapper_id": "The ID of the tap, in kebab-case",
+    "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
+    "serialization_method": "The serialization method to use for loading data",
+    "include_ci_files": "Whether to include CI files for a common CI services",
+    "license": "The license for the project"
+  }
 }

cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml

@@ -7,7 +7,7 @@ name = "{{cookiecutter.target_id}}"
 version = "0.0.1"
 description = "`{{cookiecutter.target_id}}` is a Singer target for {{cookiecutter.destination_name}}, built with the Meltano Singer SDK."
 readme = "README.md"
-authors = ["{{ cookiecutter.admin_name }}"]
+authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"]
 keywords = [
     "ELT",
     "{{cookiecutter.destination_name}}",
@@ -20,16 +20,16 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = "<3.12,>=3.7.1"
-singer-sdk = { version="^0.31.0" }
-fs-s3fs = { version = "^1.1.1", optional = true }
+python = ">=3.7.1,<4"
+singer-sdk = { version="~=0.32.0b3" }
+fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.serialization_method != "SQL" %}
-requests = "^2.31.0"
+requests = "~=2.31.0"
 {%- endif %}
 
 [tool.poetry.dev-dependencies]
-pytest = "^7.4.0"
-singer-sdk = { version="^0.31.0", extras = ["testing"] }
+pytest = ">=7.4.0"
+singer-sdk = { version="~=0.32.0b3", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/sinks.py

@@ -35,6 +35,7 @@ class {{ cookiecutter.destination_name }}Connector(SQLConnector):
     allow_column_rename: bool = True  # Whether RENAME COLUMN is supported.
     allow_column_alter: bool = False  # Whether altering column types is supported.
     allow_merge_upsert: bool = False  # Whether MERGE UPSERT is supported.
+    allow_overwrite: bool = False  # Whether overwrite load method is supported.
     allow_temp_tables: bool = True  # Whether temp tables are supported.
 
     def get_sqlalchemy_url(self, config: dict) -> str: