Merge branch 'main' into http-connector

.pre-commit-config.yaml

@@ -43,7 +43,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.280
+  rev: v0.0.282
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
@@ -64,7 +64,7 @@ repos:
         )$
 
 - repo: https://github.com/pycqa/flake8
-  rev: 6.0.0
+  rev: 6.1.0
   hooks:
   - id: flake8
     additional_dependencies:

cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml

@@ -14,24 +14,24 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.23.0
+  rev: 0.23.3
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/charliermarsh/ruff-pre-commit
-  rev: v0.0.269
+  rev: v0.0.282
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
 
 - repo: https://github.com/psf/black
-  rev: 23.3.0
+  rev: 23.7.0
   hooks:
   - id: black
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.3.0
+  rev: v1.4.1
   hooks:
   - id: mypy
     additional_dependencies:

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%}

@@ -159,7 +159,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
     def get_url_params(
         self,
         context: dict | None,  # noqa: ARG002
-        next_page_token: Any | None,
+        next_page_token: Any | None,  # noqa: ANN401
     ) -> dict[str, Any]:
         """Return a dictionary of values to be used in URL parameterization.
 
@@ -181,7 +181,7 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
     def prepare_request_payload(
         self,
         context: dict | None,  # noqa: ARG002
-        next_page_token: Any | None,  # noqa: ARG002
+        next_page_token: Any | None,  # noqa: ARG002, ANN401
     ) -> dict | None:
         """Prepare the data payload for the REST API request.
 

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%}

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import typing as t
 from pathlib import Path
 
 from singer_sdk import typing as th  # JSON Schema typing helpers
@@ -54,7 +55,7 @@ class UsersStream({{ cookiecutter.source_name }}Stream):
             ),
         ),
     ).to_dict()
-    primary_keys = ["id"]
+    primary_keys: t.ClassVar[list[str]] = ["id"]
     replication_key = None
     graphql_query = """
         users {
@@ -81,7 +82,7 @@ class GroupsStream({{ cookiecutter.source_name }}Stream):
         th.Property("id", th.StringType),
         th.Property("modified", th.DateTimeType),
     ).to_dict()
-    primary_keys = ["id"]
+    primary_keys: t.ClassVar[list[str]] = ["id"]
     replication_key = "modified"
     graphql_query = """
         groups {
@@ -104,7 +105,7 @@ class UsersStream({{ cookiecutter.source_name }}Stream):
 {%- if cookiecutter.stream_type == "REST" %}
     path = "/users"
 {%- endif %}
-    primary_keys = ["id"]
+    primary_keys: t.ClassVar[list[str]] = ["id"]
     replication_key = None
     # Optionally, you may also use `schema_filepath` in place of `schema`:
     # schema_filepath = SCHEMAS_DIR / "users.json"  # noqa: ERA001
@@ -143,7 +144,7 @@ class GroupsStream({{ cookiecutter.source_name }}Stream):
 {%- if cookiecutter.stream_type == "REST" %}
     path = "/groups"
 {%- endif %}
-    primary_keys = ["id"]
+    primary_keys: t.ClassVar[list[str]] = ["id"]
     replication_key = "modified"
     schema = th.PropertiesList(
         th.Property("name", th.StringType),

cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml

@@ -14,24 +14,24 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.23.0
+  rev: 0.23.3
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/charliermarsh/ruff-pre-commit
-  rev: v0.0.269
+  rev: v0.0.282
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
 
 - repo: https://github.com/psf/black
-  rev: 23.3.0
+  rev: 23.7.0
   hooks:
   - id: black
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.3.0
+  rev: v1.4.1
   hooks:
   - id: mypy
     additional_dependencies:

noxfile.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import glob
 import os
 import shutil
 import sys
@@ -23,9 +22,11 @@
 
 RUFF_OVERRIDES = """\
 extend = "./pyproject.toml"
-extend-ignore = ["TD002", "TD003"]
+extend-ignore = ["TD002", "TD003", "FIX002"]
 """
 
+COOKIECUTTER_REPLAY_FILES = list(Path("./e2e-tests/cookiecutters").glob("*.json"))
+
 package = "singer_sdk"
 python_versions = ["3.11", "3.10", "3.9", "3.8", "3.7"]
 main_python_version = "3.10"
@@ -186,7 +187,7 @@ def docs_serve(session: Session) -> None:
     session.run("sphinx-autobuild", *args)
 
 
-@nox.parametrize("replay_file_path", glob.glob("./e2e-tests/cookiecutters/*.json"))
+@nox.parametrize("replay_file_path", COOKIECUTTER_REPLAY_FILES)
 @session(python=main_python_version)
 def test_cookiecutter(session: Session, replay_file_path) -> None:
     """Uses the tap template to build an empty cookiecutter.

singer_sdk/helpers/capabilities.py

@@ -99,6 +99,13 @@
         description="The default target database schema name to use for all streams.",
     ),
 ).to_dict()
+ADD_RECORD_METADATA_CONFIG = PropertiesList(
+    Property(
+        "add_record_metadata",
+        BooleanType(),
+        description="Add metadata to records.",
+    ),
+).to_dict()
 
 
 class DeprecatedEnum(Enum):