diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 61a0e143a..a6f131479 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -32,6 +32,7 @@ body: description: Version of Python you are using options: - "NA" + - "3.13" - "3.12" - "3.11" - "3.10" diff --git a/.github/workflows/api-changes.yml b/.github/workflows/api-changes.yml index 9cbae5843..108d5b6f0 100644 --- a/.github/workflows/api-changes.yml +++ b/.github/workflows/api-changes.yml @@ -30,7 +30,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: 3.12 + python-version: 3.x - name: Install tools env: diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 122e9d090..41497ffb6 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.12 + python-version: 3.x architecture: x64 - name: Install poetry diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index ef2d5e4d5..64c289d4c 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -24,14 +24,8 @@ env: jobs: lint: - name: Cookiecutter E2E Python ${{ matrix.python-version }} / ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - include: - - { python-version: "3.12", os: "ubuntu-latest" } - + name: Cookiecutter E2E Python + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Upgrade pip @@ -50,8 +44,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - architecture: x64 + python-version: 3.x cache: 'pip' cache-dependency-path: 'poetry.lock' @@ -69,12 +62,12 @@ jobs: - name: Run Nox run: | - nox --python=${{ matrix.python-version }} --session=test_cookiecutter + nox --session=test_cookiecutter - uses: actions/upload-artifact@v4 if: always() with: - name: cookiecutter-${{ matrix.os }}-py${{ matrix.python-version }} + name: cookiecutter-ubuntu-latest-py3x path: | /tmp/tap-* /tmp/target-* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 89ab95710..29432a002 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,11 +48,17 @@ jobs: matrix: session: [tests] os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" sqlalchemy: ["2"] include: - - { session: tests, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "1" } - - { session: doctest, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" } + - { session: tests, python-version: "3.13", os: "ubuntu-latest", sqlalchemy: "1" } + - { session: doctest, python-version: "3.13", os: "ubuntu-latest", sqlalchemy: "2" } - { session: mypy, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" } - { session: deps, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" } @@ -64,6 +70,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Upgrade pip env: @@ -88,6 +95,8 @@ jobs: - name: Run Nox env: SQLALCHEMY_VERSION: ${{ matrix.sqlalchemy }} + PIP_PRE: "1" + UV_PRERELEASE: allow run: | nox --verbose @@ -134,6 +143,9 @@ jobs: nox --version - name: Run Nox + env: + PIP_PRE: "1" + UV_PRERELEASE: allow run: | nox -- -m "external" @@ -147,7 +159,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.x' - name: Upgrade pip env: diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 7f56cfba6..5d94ddf8d 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -47,8 +47,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.12" - architecture: x64 + python-version: "3.x" - name: Bump version id: cz-bump diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18f9fecda..5796aba15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.3 + rev: 0.29.4 hooks: - id: check-dependabot - id: check-github-workflows diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index 919a92dc6..fee771429 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -20,6 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] license = "Apache-2.0" {%- if cookiecutter.variant != "None (Skip)" %} diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini index fb5b1cf87..90c122a54 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini @@ -1,7 +1,7 @@ # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy [tox] -envlist = py3{9,10,11,12} +envlist = py3{9,10,11,12,13} requires = tox>=4.19 diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 38ef2b94f..19f1ee931 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] license = "Apache-2.0" {%- if cookiecutter.variant != "None (Skip)" %} diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini index fb5b1cf87..90c122a54 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini @@ -1,7 +1,7 @@ # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy [tox] -envlist = py3{9,10,11,12} +envlist = py3{9,10,11,12,13} requires = tox>=4.19 diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index 27cb43531..4deff5c78 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] license = "Apache-2.0" {%- if cookiecutter.variant != "None (Skip)" %} diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini b/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini index fb5b1cf87..90c122a54 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini @@ -1,7 +1,7 @@ # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy [tox] -envlist = py3{9,10,11,12} +envlist = py3{9,10,11,12,13} requires = tox>=4.19 diff --git a/docs/implementation/cli.md b/docs/implementation/cli.md index 94077fc5d..92d77fa37 100644 --- a/docs/implementation/cli.md +++ b/docs/implementation/cli.md @@ -68,9 +68,19 @@ values from environment variables, and from a `.env` file if present within the working directory, which match the exact name of a setting, along with a prefix determined by the plugin name. -> For example: For a sample plugin named `tap-my-example` and settings named "username" and "access_key", the SDK will automatically scrape -> the settings from environment variables `TAP_MY_EXAMPLE_USERNAME` and -> `TAP_MY_EXAMPLE_ACCESS_KEY`, if they exist. +```{note} +For example, for a sample plugin named `tap-my-example` and settings named `username` and `access_key`, the SDK will automatically scrape +the settings from environment variables `TAP_MY_EXAMPLE_USERNAME` and +`TAP_MY_EXAMPLE_ACCESS_KEY` respectively, if they exist. +``` + +The following value types are automatically cast to the appropriate Python type: + +- integer (e.g. `TAP_MY_EXAMPLE_PORT=5432`) +- boolean (e.g. `TAP_MY_EXAMPLE_DEBUG=true`) +- JSON arrays (e.g. `TAP_MY_EXAMPLE_ARRAY='["a", "b", "c"]'`) +- JSON objects (e.g. `TAP_MY_EXAMPLE_OBJECT='{"key": "value"}'`) + ## Tap-Specific CLI Options diff --git a/noxfile.py b/noxfile.py index 1641901e9..3a94fe9fa 100644 --- a/noxfile.py +++ b/noxfile.py @@ -6,6 +6,7 @@ import shutil import sys import tempfile +import typing as t from pathlib import Path import nox @@ -23,7 +24,14 @@ COOKIECUTTER_REPLAY_FILES = list(Path("./e2e-tests/cookiecutters").glob("*.json")) package = "singer_sdk" -python_versions = ["3.12", "3.11", "3.10", "3.9", "3.8"] +python_versions = [ + "3.13", + "3.12", + "3.11", + "3.10", + "3.9", + "3.8", +] main_python_version = "3.12" locations = "singer_sdk", "tests", "noxfile.py", "docs/conf.py" nox.options.sessions = ( @@ -35,7 +43,7 @@ ) poetry_config = nox.project.load_toml("pyproject.toml")["tool"]["poetry"] -test_dependencies = poetry_config["group"]["dev"]["dependencies"].keys() +test_dependencies: dict[str, t.Any] = poetry_config["group"]["dev"]["dependencies"] typing_dependencies = poetry_config["group"]["typing"]["dependencies"].keys() @@ -53,7 +61,18 @@ def mypy(session: nox.Session) -> None: @nox.session(python=python_versions) def tests(session: nox.Session) -> None: """Execute pytest tests and compute coverage.""" - session.install(".[faker,jwt,parquet,s3]") + extras = [ + "faker", + "jwt", + "parquet", + "s3", + ] + + if session.python == "3.13": + # https://github.com/apache/arrow/issues/43519 + extras.remove("parquet") + + session.install(f".[{','.join(extras)}]") session.install(*test_dependencies) sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION") @@ -95,7 +114,7 @@ def benches(session: nox.Session) -> None: ) -@nox.session(name="deps", python=python_versions) +@nox.session(name="deps", python=main_python_version) def dependencies(session: nox.Session) -> None: """Check issues with dependencies.""" session.install(".[s3,testing]") diff --git a/poetry.lock b/poetry.lock index 159f296c1..30ae812f5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "aiobotocore" @@ -932,13 +932,13 @@ test = ["pytest (>=6)"] [[package]] name = "faker" -version = "30.3.0" +version = "30.6.0" description = "Faker is a Python package that generates fake data for you." optional = true python-versions = ">=3.8" files = [ - {file = "Faker-30.3.0-py3-none-any.whl", hash = "sha256:e8a15fd1b0f72992b008f5ea94c70d3baa0cb51b0d5a0e899c17b1d1b23d2771"}, - {file = "faker-30.3.0.tar.gz", hash = "sha256:8760fbb34564fbb2f394345eef24aec5b8f6506b6cfcefe8195ed66dd1032bdb"}, + {file = "Faker-30.6.0-py3-none-any.whl", hash = "sha256:37b5ab951f7367ea93edb865120e9717a7a649d6a4b223f1e4a47a8a20d9e85f"}, + {file = "faker-30.6.0.tar.gz", hash = "sha256:be0e548352c1be6f6d9c982003848a0d305868f160bb1fb7f945acffc347e676"}, ] [package.dependencies] @@ -3122,60 +3122,68 @@ test = ["pytest"] [[package]] name = "sqlalchemy" -version = "2.0.35" +version = "2.0.36" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.35-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:67219632be22f14750f0d1c70e62f204ba69d28f62fd6432ba05ab295853de9b"}, - {file = "SQLAlchemy-2.0.35-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4668bd8faf7e5b71c0319407b608f278f279668f358857dbfd10ef1954ac9f90"}, - {file = "SQLAlchemy-2.0.35-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb8bea573863762bbf45d1e13f87c2d2fd32cee2dbd50d050f83f87429c9e1ea"}, - {file = "SQLAlchemy-2.0.35-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f552023710d4b93d8fb29a91fadf97de89c5926c6bd758897875435f2a939f33"}, - {file = "SQLAlchemy-2.0.35-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:016b2e665f778f13d3c438651dd4de244214b527a275e0acf1d44c05bc6026a9"}, - {file = "SQLAlchemy-2.0.35-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7befc148de64b6060937231cbff8d01ccf0bfd75aa26383ffdf8d82b12ec04ff"}, - {file = "SQLAlchemy-2.0.35-cp310-cp310-win32.whl", hash = "sha256:22b83aed390e3099584b839b93f80a0f4a95ee7f48270c97c90acd40ee646f0b"}, - {file = "SQLAlchemy-2.0.35-cp310-cp310-win_amd64.whl", hash = "sha256:a29762cd3d116585278ffb2e5b8cc311fb095ea278b96feef28d0b423154858e"}, - {file = "SQLAlchemy-2.0.35-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e21f66748ab725ade40fa7af8ec8b5019c68ab00b929f6643e1b1af461eddb60"}, - {file = "SQLAlchemy-2.0.35-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8a6219108a15fc6d24de499d0d515c7235c617b2540d97116b663dade1a54d62"}, - {file = "SQLAlchemy-2.0.35-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:042622a5306c23b972192283f4e22372da3b8ddf5f7aac1cc5d9c9b222ab3ff6"}, - {file = "SQLAlchemy-2.0.35-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:627dee0c280eea91aed87b20a1f849e9ae2fe719d52cbf847c0e0ea34464b3f7"}, - {file = "SQLAlchemy-2.0.35-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4fdcd72a789c1c31ed242fd8c1bcd9ea186a98ee8e5408a50e610edfef980d71"}, - {file = "SQLAlchemy-2.0.35-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:89b64cd8898a3a6f642db4eb7b26d1b28a497d4022eccd7717ca066823e9fb01"}, - {file = "SQLAlchemy-2.0.35-cp311-cp311-win32.whl", hash = "sha256:6a93c5a0dfe8d34951e8a6f499a9479ffb9258123551fa007fc708ae2ac2bc5e"}, - {file = "SQLAlchemy-2.0.35-cp311-cp311-win_amd64.whl", hash = "sha256:c68fe3fcde03920c46697585620135b4ecfdfc1ed23e75cc2c2ae9f8502c10b8"}, - {file = "SQLAlchemy-2.0.35-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:eb60b026d8ad0c97917cb81d3662d0b39b8ff1335e3fabb24984c6acd0c900a2"}, - {file = "SQLAlchemy-2.0.35-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6921ee01caf375363be5e9ae70d08ce7ca9d7e0e8983183080211a062d299468"}, - {file = "SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cdf1a0dbe5ced887a9b127da4ffd7354e9c1a3b9bb330dce84df6b70ccb3a8d"}, - {file = "SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93a71c8601e823236ac0e5d087e4f397874a421017b3318fd92c0b14acf2b6db"}, - {file = "SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e04b622bb8a88f10e439084486f2f6349bf4d50605ac3e445869c7ea5cf0fa8c"}, - {file = "SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1b56961e2d31389aaadf4906d453859f35302b4eb818d34a26fab72596076bb8"}, - {file = "SQLAlchemy-2.0.35-cp312-cp312-win32.whl", hash = "sha256:0f9f3f9a3763b9c4deb8c5d09c4cc52ffe49f9876af41cc1b2ad0138878453cf"}, - {file = "SQLAlchemy-2.0.35-cp312-cp312-win_amd64.whl", hash = "sha256:25b0f63e7fcc2a6290cb5f7f5b4fc4047843504983a28856ce9b35d8f7de03cc"}, - {file = "SQLAlchemy-2.0.35-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f021d334f2ca692523aaf7bbf7592ceff70c8594fad853416a81d66b35e3abf9"}, - {file = "SQLAlchemy-2.0.35-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05c3f58cf91683102f2f0265c0db3bd3892e9eedabe059720492dbaa4f922da1"}, - {file = "SQLAlchemy-2.0.35-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:032d979ce77a6c2432653322ba4cbeabf5a6837f704d16fa38b5a05d8e21fa00"}, - {file = "SQLAlchemy-2.0.35-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:2e795c2f7d7249b75bb5f479b432a51b59041580d20599d4e112b5f2046437a3"}, - {file = "SQLAlchemy-2.0.35-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:cc32b2990fc34380ec2f6195f33a76b6cdaa9eecf09f0c9404b74fc120aef36f"}, - {file = "SQLAlchemy-2.0.35-cp37-cp37m-win32.whl", hash = "sha256:9509c4123491d0e63fb5e16199e09f8e262066e58903e84615c301dde8fa2e87"}, - {file = "SQLAlchemy-2.0.35-cp37-cp37m-win_amd64.whl", hash = "sha256:3655af10ebcc0f1e4e06c5900bb33e080d6a1fa4228f502121f28a3b1753cde5"}, - {file = "SQLAlchemy-2.0.35-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4c31943b61ed8fdd63dfd12ccc919f2bf95eefca133767db6fbbd15da62078ec"}, - {file = "SQLAlchemy-2.0.35-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a62dd5d7cc8626a3634208df458c5fe4f21200d96a74d122c83bc2015b333bc1"}, - {file = "SQLAlchemy-2.0.35-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0630774b0977804fba4b6bbea6852ab56c14965a2b0c7fc7282c5f7d90a1ae72"}, - {file = "SQLAlchemy-2.0.35-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d625eddf7efeba2abfd9c014a22c0f6b3796e0ffb48f5d5ab106568ef01ff5a"}, - {file = "SQLAlchemy-2.0.35-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ada603db10bb865bbe591939de854faf2c60f43c9b763e90f653224138f910d9"}, - {file = "SQLAlchemy-2.0.35-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c41411e192f8d3ea39ea70e0fae48762cd11a2244e03751a98bd3c0ca9a4e936"}, - {file = "SQLAlchemy-2.0.35-cp38-cp38-win32.whl", hash = "sha256:d299797d75cd747e7797b1b41817111406b8b10a4f88b6e8fe5b5e59598b43b0"}, - {file = "SQLAlchemy-2.0.35-cp38-cp38-win_amd64.whl", hash = "sha256:0375a141e1c0878103eb3d719eb6d5aa444b490c96f3fedab8471c7f6ffe70ee"}, - {file = "SQLAlchemy-2.0.35-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ccae5de2a0140d8be6838c331604f91d6fafd0735dbdcee1ac78fc8fbaba76b4"}, - {file = "SQLAlchemy-2.0.35-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a275a806f73e849e1c309ac11108ea1a14cd7058577aba962cd7190e27c9e3c"}, - {file = "SQLAlchemy-2.0.35-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:732e026240cdd1c1b2e3ac515c7a23820430ed94292ce33806a95869c46bd139"}, - {file = "SQLAlchemy-2.0.35-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:890da8cd1941fa3dab28c5bac3b9da8502e7e366f895b3b8e500896f12f94d11"}, - {file = "SQLAlchemy-2.0.35-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c0d8326269dbf944b9201911b0d9f3dc524d64779a07518199a58384c3d37a44"}, - {file = "SQLAlchemy-2.0.35-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b76d63495b0508ab9fc23f8152bac63205d2a704cd009a2b0722f4c8e0cba8e0"}, - {file = "SQLAlchemy-2.0.35-cp39-cp39-win32.whl", hash = "sha256:69683e02e8a9de37f17985905a5eca18ad651bf592314b4d3d799029797d0eb3"}, - {file = "SQLAlchemy-2.0.35-cp39-cp39-win_amd64.whl", hash = "sha256:aee110e4ef3c528f3abbc3c2018c121e708938adeeff9006428dd7c8555e9b3f"}, - {file = "SQLAlchemy-2.0.35-py3-none-any.whl", hash = "sha256:2ab3f0336c0387662ce6221ad30ab3a5e6499aab01b9790879b6578fd9b8faa1"}, - {file = "sqlalchemy-2.0.35.tar.gz", hash = "sha256:e11d7ea4d24f0a262bccf9a7cd6284c976c5369dac21db237cff59586045ab9f"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59b8f3adb3971929a3e660337f5dacc5942c2cdb760afcabb2614ffbda9f9f72"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37350015056a553e442ff672c2d20e6f4b6d0b2495691fa239d8aa18bb3bc908"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8318f4776c85abc3f40ab185e388bee7a6ea99e7fa3a30686580b209eaa35c08"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c245b1fbade9c35e5bd3b64270ab49ce990369018289ecfde3f9c318411aaa07"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:69f93723edbca7342624d09f6704e7126b152eaed3cdbb634cb657a54332a3c5"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f9511d8dd4a6e9271d07d150fb2f81874a3c8c95e11ff9af3a2dfc35fe42ee44"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-win32.whl", hash = "sha256:c3f3631693003d8e585d4200730616b78fafd5a01ef8b698f6967da5c605b3fa"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-win_amd64.whl", hash = "sha256:a86bfab2ef46d63300c0f06936bd6e6c0105faa11d509083ba8f2f9d237fb5b5"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fd3a55deef00f689ce931d4d1b23fa9f04c880a48ee97af488fd215cf24e2a6c"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f5e9cd989b45b73bd359f693b935364f7e1f79486e29015813c338450aa5a71"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ddd9db6e59c44875211bc4c7953a9f6638b937b0a88ae6d09eb46cced54eff"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2519f3a5d0517fc159afab1015e54bb81b4406c278749779be57a569d8d1bb0d"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59b1ee96617135f6e1d6f275bbe988f419c5178016f3d41d3c0abb0c819f75bb"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39769a115f730d683b0eb7b694db9789267bcd027326cccc3125e862eb03bfd8"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-win32.whl", hash = "sha256:66bffbad8d6271bb1cc2f9a4ea4f86f80fe5e2e3e501a5ae2a3dc6a76e604e6f"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-win_amd64.whl", hash = "sha256:23623166bfefe1487d81b698c423f8678e80df8b54614c2bf4b4cfcd7c711959"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7b64e6ec3f02c35647be6b4851008b26cff592a95ecb13b6788a54ef80bbdd4"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:46331b00096a6db1fdc052d55b101dbbfc99155a548e20a0e4a8e5e4d1362855"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdf3386a801ea5aba17c6410dd1dc8d39cf454ca2565541b5ac42a84e1e28f53"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9dfa18ff2a67b09b372d5db8743c27966abf0e5344c555d86cc7199f7ad83a"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:90812a8933df713fdf748b355527e3af257a11e415b613dd794512461eb8a686"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1bc330d9d29c7f06f003ab10e1eaced295e87940405afe1b110f2eb93a233588"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-win32.whl", hash = "sha256:79d2e78abc26d871875b419e1fd3c0bca31a1cb0043277d0d850014599626c2e"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-win_amd64.whl", hash = "sha256:b544ad1935a8541d177cb402948b94e871067656b3a0b9e91dbec136b06a2ff5"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5cc79df7f4bc3d11e4b542596c03826063092611e481fcf1c9dfee3c94355ef"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3c01117dd36800f2ecaa238c65365b7b16497adc1522bf84906e5710ee9ba0e8"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bc633f4ee4b4c46e7adcb3a9b5ec083bf1d9a97c1d3854b92749d935de40b9b"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e46ed38affdfc95d2c958de328d037d87801cfcbea6d421000859e9789e61c2"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b2985c0b06e989c043f1dc09d4fe89e1616aadd35392aea2844f0458a989eacf"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a121d62ebe7d26fec9155f83f8be5189ef1405f5973ea4874a26fab9f1e262c"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-win32.whl", hash = "sha256:0572f4bd6f94752167adfd7c1bed84f4b240ee6203a95e05d1e208d488d0d436"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-win_amd64.whl", hash = "sha256:8c78ac40bde930c60e0f78b3cd184c580f89456dd87fc08f9e3ee3ce8765ce88"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:be9812b766cad94a25bc63bec11f88c4ad3629a0cec1cd5d4ba48dc23860486b"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50aae840ebbd6cdd41af1c14590e5741665e5272d2fee999306673a1bb1fdb4d"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4557e1f11c5f653ebfdd924f3f9d5ebfc718283b0b9beebaa5dd6b77ec290971"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07b441f7d03b9a66299ce7ccf3ef2900abc81c0db434f42a5694a37bd73870f2"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:28120ef39c92c2dd60f2721af9328479516844c6b550b077ca450c7d7dc68575"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-win32.whl", hash = "sha256:b81ee3d84803fd42d0b154cb6892ae57ea6b7c55d8359a02379965706c7efe6c"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-win_amd64.whl", hash = "sha256:f942a799516184c855e1a32fbc7b29d7e571b52612647866d4ec1c3242578fcb"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3d6718667da04294d7df1670d70eeddd414f313738d20a6f1d1f379e3139a545"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:72c28b84b174ce8af8504ca28ae9347d317f9dba3999e5981a3cd441f3712e24"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b11d0cfdd2b095e7b0686cf5fabeb9c67fae5b06d265d8180715b8cfa86522e3"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e32092c47011d113dc01ab3e1d3ce9f006a47223b18422c5c0d150af13a00687"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:6a440293d802d3011028e14e4226da1434b373cbaf4a4bbb63f845761a708346"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c54a1e53a0c308a8e8a7dffb59097bff7facda27c70c286f005327f21b2bd6b1"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-win32.whl", hash = "sha256:1e0d612a17581b6616ff03c8e3d5eff7452f34655c901f75d62bd86449d9750e"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-win_amd64.whl", hash = "sha256:8958b10490125124463095bbdadda5aa22ec799f91958e410438ad6c97a7b793"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc022184d3e5cacc9579e41805a681187650e170eb2fd70e28b86192a479dcaa"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b817d41d692bf286abc181f8af476c4fbef3fd05e798777492618378448ee689"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4e46a888b54be23d03a89be510f24a7652fe6ff660787b96cd0e57a4ebcb46d"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4ae3005ed83f5967f961fd091f2f8c5329161f69ce8480aa8168b2d7fe37f06"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:03e08af7a5f9386a43919eda9de33ffda16b44eb11f3b313e6822243770e9763"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3dbb986bad3ed5ceaf090200eba750b5245150bd97d3e67343a3cfed06feecf7"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-win32.whl", hash = "sha256:9fe53b404f24789b5ea9003fc25b9a3988feddebd7e7b369c8fac27ad6f52f28"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-win_amd64.whl", hash = "sha256:af148a33ff0349f53512a049c6406923e4e02bf2f26c5fb285f143faf4f0e46a"}, + {file = "SQLAlchemy-2.0.36-py3-none-any.whl", hash = "sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e"}, + {file = "sqlalchemy-2.0.36.tar.gz", hash = "sha256:7f2767680b6d2398aea7082e45a774b2b0767b5c8d8ffb9c8b683088ea9b29c5"}, ] [package.dependencies] @@ -3188,7 +3196,7 @@ aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5,!=1.1.10)"] mssql = ["pyodbc"] mssql-pymssql = ["pymssql"] mssql-pyodbc = ["pyodbc"] @@ -3349,13 +3357,13 @@ types-urllib3 = "*" [[package]] name = "types-requests" -version = "2.32.0.20240914" +version = "2.32.0.20241016" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" files = [ - {file = "types-requests-2.32.0.20240914.tar.gz", hash = "sha256:2850e178db3919d9bf809e434eef65ba49d0e7e33ac92d588f4a5e295fffd405"}, - {file = "types_requests-2.32.0.20240914-py3-none-any.whl", hash = "sha256:59c2f673eb55f32a99b2894faf6020e1a9f4a402ad0f192bfee0b64469054310"}, + {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, + {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, ] [package.dependencies] @@ -3566,13 +3574,13 @@ tests-strict = ["pytest (==4.6.0)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)"] [[package]] name = "xmltodict" -version = "0.14.1" +version = "0.14.2" description = "Makes working with XML feel like you are working with JSON" optional = false python-versions = ">=3.6" files = [ - {file = "xmltodict-0.14.1-py2.py3-none-any.whl", hash = "sha256:3ef4a7b71c08f19047fcbea572e1d7f4207ab269da1565b5d40e9823d3894e63"}, - {file = "xmltodict-0.14.1.tar.gz", hash = "sha256:338c8431e4fc554517651972d62f06958718f6262b04316917008e8fd677a6b0"}, + {file = "xmltodict-0.14.2-py2.py3-none-any.whl", hash = "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac"}, + {file = "xmltodict-0.14.2.tar.gz", hash = "sha256:201e7c28bb210e374999d1dde6382923ab0ed1a8a5faeece48ab525b7810a553"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 329705b8a..e137cdd8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,6 +173,8 @@ filterwarnings = [ "ignore:No records were available to test:UserWarning", # https://github.com/meltano/sdk/issues/1354 "ignore:The function singer_sdk.testing.get_standard_tap_tests is deprecated:DeprecationWarning", + # TODO: Address this SQLite warning in Python 3.13+ + "ignore::ResourceWarning", ] log_cli_level = "INFO" markers = [ diff --git a/samples/sample_tap_dummy_json/pyproject.toml b/samples/sample_tap_dummy_json/pyproject.toml index c72347748..4542c6448 100644 --- a/samples/sample_tap_dummy_json/pyproject.toml +++ b/samples/sample_tap_dummy_json/pyproject.toml @@ -15,6 +15,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] license = "Apache-2.0" diff --git a/samples/sample_tap_dummy_json/ruff.toml b/samples/sample_tap_dummy_json/ruff.toml index dd2f13459..c6af4cfef 100644 --- a/samples/sample_tap_dummy_json/ruff.toml +++ b/samples/sample_tap_dummy_json/ruff.toml @@ -1,5 +1,5 @@ src = ["tap_dummyjson"] -target-version = "py38" +target-version = "py39" [lint] ignore = [ diff --git a/samples/sample_tap_dummy_json/tox.ini b/samples/sample_tap_dummy_json/tox.ini index 6be1c116a..8b89a4ef1 100644 --- a/samples/sample_tap_dummy_json/tox.ini +++ b/samples/sample_tap_dummy_json/tox.ini @@ -1,7 +1,7 @@ # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy [tox] -envlist = py{38,39,310,311,312} +envlist = py3{9,10,11,12,13} isolated_build = true [testenv] @@ -13,7 +13,7 @@ commands = [testenv:pytest] # Run the python tests. # To execute, run `tox -e pytest` -envlist = py{38,39,310,311,312} +envlist = py3{8,9,10,11,12,313} commands = poetry install -v poetry run pytest diff --git a/samples/sample_target_parquet/parquet_target_sink.py b/samples/sample_target_parquet/parquet_target_sink.py index e98dca2b1..fbdd13cd1 100644 --- a/samples/sample_target_parquet/parquet_target_sink.py +++ b/samples/sample_target_parquet/parquet_target_sink.py @@ -4,11 +4,14 @@ import typing as t -import pyarrow as pa -import pyarrow.parquet as pq - from singer_sdk.sinks import BatchSink +try: + import pyarrow as pa + import pyarrow.parquet as pq +except ImportError: + pass + def json_schema_to_arrow(schema: dict[str, t.Any]) -> pa.Schema: """Convert a JSON Schema to an Arrow schema. diff --git a/singer_sdk/about.py b/singer_sdk/about.py index ab3062295..4b0cc1b8d 100644 --- a/singer_sdk/about.py +++ b/singer_sdk/about.py @@ -24,7 +24,7 @@ # Keep these in sync with the supported Python versions in pyproject.toml _PY_MIN_VERSION = 8 -_PY_MAX_VERSION = 12 +_PY_MAX_VERSION = 13 def _get_min_version(specifiers: SpecifierSet) -> int: diff --git a/singer_sdk/configuration/_dict_config.py b/singer_sdk/configuration/_dict_config.py index fd8217f01..af1d19f29 100644 --- a/singer_sdk/configuration/_dict_config.py +++ b/singer_sdk/configuration/_dict_config.py @@ -10,11 +10,21 @@ from dotenv import find_dotenv from dotenv.main import DotEnv -from singer_sdk.helpers._typing import is_string_array_type -from singer_sdk.helpers._util import read_json_file +from singer_sdk.helpers import _typing +from singer_sdk.helpers._util import load_json, read_json_file logger = logging.getLogger(__name__) +TRUTHY = ("true", "1", "yes", "on") + + +def _parse_array(value: str) -> list[str]: + return load_json(value) # type: ignore[return-value] + + +def _legacy_parse_array_of_strings(value: str) -> list[str]: + return value.split(",") + def parse_environment_config( config_schema: dict[str, t.Any], @@ -29,9 +39,6 @@ def parse_environment_config( dotenv_path: Path to a .env file. If None, will try to find one in increasingly higher folders. - Raises: - ValueError: If an un-parsable setting is found. - Returns: A configuration dictionary. """ @@ -43,7 +50,7 @@ def parse_environment_config( logger.debug("Loading configuration from %s", dotenv_path) DotEnv(dotenv_path).set_as_environment_variables() - for config_key in config_schema["properties"]: + for config_key, schema in config_schema.get("properties", {}).items(): env_var_name = prefix + config_key.upper().replace("-", "_") if env_var_name in os.environ: env_var_value = os.environ[env_var_name] @@ -52,15 +59,25 @@ def parse_environment_config( config_key, env_var_name, ) - if is_string_array_type(config_schema["properties"][config_key]): - if env_var_value[0] == "[" and env_var_value[-1] == "]": - msg = ( - "A bracketed list was detected in the environment variable " - f"'{env_var_name}'. This syntax is no longer supported. Please " - "remove the brackets and try again." + if _typing.is_integer_type(schema): + result[config_key] = int(env_var_value) + elif _typing.is_boolean_type(schema): + result[config_key] = env_var_value.lower() in TRUTHY + elif _typing.is_string_array_type(schema): + try: + result[config_key] = _parse_array(env_var_value) + except Exception: # noqa: BLE001 + # TODO(edgarrmondragon): Make this a deprecation warning. + # https://github.com/meltano/sdk/issues/2724 + logger.warning( + "Parsing array of the form 'x,y,z' is deprecated and will be " + "removed in future versions.", ) - raise ValueError(msg) - result[config_key] = env_var_value.split(",") + result[config_key] = _legacy_parse_array_of_strings(env_var_value) + elif _typing.is_array_type(schema): + result[config_key] = _parse_array(env_var_value) + elif _typing.is_object_type(schema): + result[config_key] = load_json(env_var_value) else: result[config_key] = env_var_value return result diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 1e7370fd9..b02ecb512 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -279,6 +279,17 @@ def is_boolean_type(property_schema: dict) -> bool | None: return False +def _is_exclusive_boolean_type(property_schema: dict) -> bool: + if "type" not in property_schema: + return False + + return ( + property_schema["type"] == "boolean" + or property_schema["type"] == ["boolean"] + or set(property_schema["type"]) == {"boolean", "null"} + ) + + def is_integer_type(property_schema: dict) -> bool | None: """Return true if the JSON Schema type is an integer or None if detection fails.""" if "anyOf" not in property_schema and "type" not in property_schema: @@ -523,6 +534,6 @@ def _conform_primitive_property( # noqa: PLR0911 if isinstance(elem, bytes): # for BIT value, treat 0 as False and anything else as True return elem != b"\x00" if is_boolean_type(property_schema) else elem.hex() - if is_boolean_type(property_schema): + if _is_exclusive_boolean_type(property_schema): return None if elem is None else elem != 0 return elem diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 53c379265..8b48b7074 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -206,15 +206,18 @@ def __init__( *, allowed_values: list[T] | None = None, examples: list[T] | None = None, + nullable: bool | None = None, ) -> None: """Initialize the type helper. Args: allowed_values: A list of allowed values. examples: A list of example values. + nullable: If True, the property may be null. """ self.allowed_values = allowed_values self.examples = examples + self.nullable = nullable @DefaultInstanceProperty def type_dict(self) -> dict: @@ -273,6 +276,8 @@ class StringType(JSONTypeHelper[str]): {'type': ['string'], 'enum': ['a', 'b']} >>> StringType(max_length=10).type_dict {'type': ['string'], 'maxLength': 10} + >>> StringType(max_length=10, nullable=True).type_dict + {'type': ['string', 'null'], 'maxLength': 10} """ string_format: str | None = None @@ -321,7 +326,7 @@ def type_dict(self) -> dict: A dictionary describing the type. """ result = { - "type": ["string"], + "type": ["string", "null"] if self.nullable else ["string"], **self._format, **self.extras, } @@ -460,7 +465,10 @@ def type_dict(self) -> dict: Returns: A dictionary describing the type. """ - return {"type": ["boolean"], **self.extras} + return { + "type": ["boolean", "null"] if self.nullable else ["boolean"], + **self.extras, + } class _NumericType(JSONTypeHelper[T]): @@ -507,7 +515,12 @@ def type_dict(self) -> dict: Returns: A dictionary describing the type. """ - result = {"type": [self.__type_name__], **self.extras} + result = { + "type": [self.__type_name__, "null"] + if self.nullable + else [self.__type_name__], + **self.extras, + } if self.minimum is not None: result["minimum"] = self.minimum @@ -592,7 +605,11 @@ def type_dict(self) -> dict: # type: ignore[override] Returns: A dictionary describing the type. """ - return {"type": "array", "items": self.wrapped_type.type_dict, **self.extras} + return { + "type": ["array", "null"] if self.nullable else "array", + "items": self.wrapped_type.type_dict, + **self.extras, + } class AnyType(JSONTypeHelper): @@ -835,7 +852,10 @@ def type_dict(self) -> dict: # type: ignore[override] merged_props.update(w.to_dict()) if not w.optional: required.append(w.name) - result: dict[str, t.Any] = {"type": "object", "properties": merged_props} + result: dict[str, t.Any] = { + "type": ["object", "null"] if self.nullable else "object", + "properties": merged_props, + } if required: result["required"] = required diff --git a/tests/contrib/test_batch_encoder_parquet.py b/tests/contrib/test_batch_encoder_parquet.py index 0318e41d3..90af43619 100644 --- a/tests/contrib/test_batch_encoder_parquet.py +++ b/tests/contrib/test_batch_encoder_parquet.py @@ -2,8 +2,11 @@ from __future__ import annotations +import sys import typing as t +import pytest + from singer_sdk.contrib.batch_encoder_parquet import ParquetBatcher from singer_sdk.helpers._batch import BatchConfig, ParquetEncoding, StorageTarget @@ -11,6 +14,11 @@ from pathlib import Path +@pytest.mark.xfail( + sys.version_info >= (3, 13), + reason="Parquet not supported on Python 3.13 due to PyArrow incompatibility", + strict=True, +) def test_batcher(tmp_path: Path) -> None: root = tmp_path.joinpath("batches") root.mkdir() @@ -30,6 +38,11 @@ def test_batcher(tmp_path: Path) -> None: assert batches[0][0].endswith(".parquet") +@pytest.mark.xfail( + sys.version_info >= (3, 13), + reason="Parquet not supported on Python 3.13 due to PyArrow incompatibility", + strict=True, +) def test_batcher_gzip(tmp_path: Path) -> None: root = tmp_path.joinpath("batches") root.mkdir() diff --git a/tests/core/configuration/test_dict_config.py b/tests/core/configuration/test_dict_config.py index 136b72e72..851f9970a 100644 --- a/tests/core/configuration/test_dict_config.py +++ b/tests/core/configuration/test_dict_config.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import logging +import typing as t from pathlib import Path import pytest @@ -11,10 +13,23 @@ parse_environment_config, ) +if t.TYPE_CHECKING: + from pytest_subtests import SubTests + CONFIG_JSONSCHEMA = th.PropertiesList( th.Property("prop1", th.StringType, required=True), th.Property("prop2", th.StringType), th.Property("prop3", th.ArrayType(th.StringType)), + th.Property("prop4", th.IntegerType), + th.Property("prop5", th.BooleanType), + th.Property("prop6", th.ArrayType(th.IntegerType)), + th.Property( + "prop7", + th.ObjectType( + th.Property("sub_prop1", th.StringType), + th.Property("sub_prop2", th.IntegerType), + ), + ), ).to_dict() @@ -36,30 +51,75 @@ def config_file2(tmpdir) -> str: return filepath -def test_get_env_var_config(monkeypatch: pytest.MonkeyPatch): +def test_get_env_var_config( + monkeypatch: pytest.MonkeyPatch, + subtests: SubTests, + caplog: pytest.LogCaptureFixture, +): """Test settings parsing from environment variables.""" with monkeypatch.context() as m: m.setenv("PLUGIN_TEST_PROP1", "hello") - m.setenv("PLUGIN_TEST_PROP3", "val1,val2") - m.setenv("PLUGIN_TEST_PROP4", "not-a-tap-setting") + m.setenv("PLUGIN_TEST_PROP3", '["val1","val2"]') + m.setenv("PLUGIN_TEST_PROP4", "123") + m.setenv("PLUGIN_TEST_PROP5", "TRUE") + m.setenv("PLUGIN_TEST_PROP6", "[1,2,3]") + m.setenv("PLUGIN_TEST_PROP7", '{"sub_prop1": "hello", "sub_prop2": 123}') + m.setenv("PLUGIN_TEST_PROP999", "not-a-tap-setting") env_config = parse_environment_config(CONFIG_JSONSCHEMA, "PLUGIN_TEST_") - assert env_config["prop1"] == "hello" - assert env_config["prop3"] == ["val1", "val2"] - assert "PROP1" not in env_config - assert "prop2" not in env_config - assert "PROP2" not in env_config - assert "prop4" not in env_config - assert "PROP4" not in env_config + + with subtests.test(msg="Parse string from environment"): + assert env_config["prop1"] == "hello" + + with subtests.test(msg="Parse array from environment"): + assert env_config["prop3"] == ["val1", "val2"] + + with subtests.test(msg="Parse integer from environment"): + assert env_config["prop4"] == 123 + + with subtests.test(msg="Parse boolean from environment"): + assert env_config["prop5"] is True + + with subtests.test(msg="Parse array of integers from environment"): + assert env_config["prop6"] == [1, 2, 3] + + with subtests.test(msg="Parse object from environment"): + assert env_config["prop7"] == {"sub_prop1": "hello", "sub_prop2": 123} + + with subtests.test(msg="Ignore non-tap setting"): + missing_props = {"PROP1", "prop2", "PROP2", "prop999", "PROP999"} + assert not set.intersection(missing_props, env_config) + + m.setenv("PLUGIN_TEST_PROP3", "val1,val2") + with subtests.test(msg="Legacy array parsing"), caplog.at_level( + logging.WARNING, + ): + parsed = parse_environment_config(CONFIG_JSONSCHEMA, "PLUGIN_TEST_") + assert parsed["prop3"] == ["val1", "val2"] + + assert any( + "Parsing array of the form 'x,y,z'" in log.message + for log in caplog.records + ) no_env_config = parse_environment_config(CONFIG_JSONSCHEMA, "PLUGIN_TEST_") - assert "prop1" not in no_env_config - assert "PROP1" not in env_config - assert "prop2" not in no_env_config - assert "PROP2" not in env_config - assert "prop3" not in no_env_config - assert "PROP3" not in env_config - assert "prop4" not in no_env_config - assert "PROP4" not in env_config + missing_props = { + "prop1", + "PROP1", + "prop2", + "PROP2", + "prop3", + "PROP3", + "prop4", + "PROP4", + "prop5", + "PROP5", + "prop6", + "PROP6", + "prop999", + "PROP999", + } + with subtests.test(msg="Ignore missing environment variables"): + assert not set.intersection(missing_props, no_env_config) def test_get_dotenv_config(tmp_path: Path): @@ -74,15 +134,6 @@ def test_get_dotenv_config(tmp_path: Path): assert dotenv_config["prop1"] == "hello" -def test_get_env_var_config_not_parsable(monkeypatch: pytest.MonkeyPatch): - """Test settings parsing from environment variables with a non-parsable value.""" - with monkeypatch.context() as m: - m.setenv("PLUGIN_TEST_PROP1", "hello") - m.setenv("PLUGIN_TEST_PROP3", '["repeated"]') - with pytest.raises(ValueError, match="A bracketed list was detected"): - parse_environment_config(CONFIG_JSONSCHEMA, "PLUGIN_TEST_") - - def test_merge_config_sources( config_file1, config_file2, @@ -91,7 +142,7 @@ def test_merge_config_sources( """Test merging multiple configuration sources.""" with monkeypatch.context() as m: m.setenv("PLUGIN_TEST_PROP1", "from-env") - m.setenv("PLUGIN_TEST_PROP4", "not-a-tap-setting") + m.setenv("PLUGIN_TEST_PROP999", "not-a-tap-setting") config = merge_config_sources( [config_file1, config_file2, "ENV"], CONFIG_JSONSCHEMA, diff --git a/tests/core/test_about.py b/tests/core/test_about.py index 70a445bb1..5d12fed19 100644 --- a/tests/core/test_about.py +++ b/tests/core/test_about.py @@ -106,8 +106,8 @@ def test_get_supported_pythons_sdk(): "specifiers,expected", [ (">=3.7,<3.12", ["3.7", "3.8", "3.9", "3.10", "3.11"]), - (">=3.7", ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]), - (">3.7", ["3.8", "3.9", "3.10", "3.11", "3.12"]), + (">=3.7", ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]), + (">3.7", ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]), (">3.7,<=3.11", ["3.8", "3.9", "3.10", "3.11"]), ], ) diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 5043c75e1..25d9c68df 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -313,7 +313,16 @@ def test_conform_primitives(): assert _conform_primitive_property(None, {"type": "boolean"}) is None assert _conform_primitive_property(0, {"type": "boolean"}) is False + assert ( + _conform_primitive_property( + 0, {"anyOf": [{"type": "boolean"}, {"type": "integer"}]} + ) + == 0 + ) + assert _conform_primitive_property(0, {"type": ["boolean", "integer"]}) == 0 assert _conform_primitive_property(1, {"type": "boolean"}) is True + assert _conform_primitive_property(1, {"type": ["boolean", "null"]}) is True + assert _conform_primitive_property(1, {"type": ["boolean"]}) is True @pytest.mark.parametrize( diff --git a/tests/samples/test_target_parquet.py b/tests/samples/test_target_parquet.py index 4be4782d5..3ec00352c 100644 --- a/tests/samples/test_target_parquet.py +++ b/tests/samples/test_target_parquet.py @@ -3,6 +3,7 @@ from __future__ import annotations import shutil +import sys import uuid from pathlib import Path @@ -23,6 +24,11 @@ ) +@pytest.mark.xfail( + sys.version_info >= (3, 13), + reason="Parquet not supported on Python 3.13 due to PyArrow incompatibility", + raises=NameError, +) class TestSampleTargetParquet(StandardTests): """Standard Target Tests.""" diff --git a/tests/samples/test_target_sqlite.py b/tests/samples/test_target_sqlite.py index 4f6d54e60..33566082f 100644 --- a/tests/samples/test_target_sqlite.py +++ b/tests/samples/test_target_sqlite.py @@ -4,6 +4,7 @@ import json import sqlite3 +import sys import typing as t from copy import deepcopy from io import StringIO @@ -367,6 +368,11 @@ def test_sqlite_process_batch_message( assert cursor.fetchone()[0] == 4 +@pytest.mark.xfail( + sys.version_info >= (3, 13), + reason="Parquet not supported on Python 3.13 due to PyArrow incompatibility", + strict=True, +) def test_sqlite_process_batch_parquet( sqlite_target_test_config: dict, sqlite_sample_target_batch: SQLiteTarget,