From 44aba7a66011b2a3a904d8afc1a7d4fbd97ba2b7 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 29 Nov 2023 00:08:27 +0100 Subject: [PATCH] MongoDB/PyMongo: Add software tests and CI configuration It needs to balance SQLAlchemy 1.x vs. 2.x throughout the toolkit test cases, because JessiQL still uses SQLAlchemy 1.x. --- .github/workflows/influxdb.yml | 2 +- .github/workflows/main.yml | 73 ++++++++++++++++++- .github/workflows/mongodb.yml | 2 +- cratedb_toolkit/adapter/pymongo/README.md | 2 +- cratedb_toolkit/adapter/pymongo/backlog.md | 3 + cratedb_toolkit/adapter/pymongo/collection.py | 3 + cratedb_toolkit/adapter/pymongo/cursor.py | 3 + cratedb_toolkit/retention/store.py | 5 +- examples/pymongo_adapter.py | 69 ++++++++++++------ pyproject.toml | 5 +- tests/adapter/test_pymongo.py | 50 +++++++++++-- tests/conftest.py | 38 ++++++++++ tests/io/influxdb/test_cli.py | 2 +- tests/io/mongodb/conftest.py | 3 + tests/io/mongodb/test_cli.py | 9 +++ tests/io/mongodb/test_integration.py | 12 ++- tests/io/test_import.py | 4 +- tests/retention/conftest.py | 3 +- tests/retention/test_cli.py | 8 +- tests/retention/test_examples.py | 2 +- tests/retention/test_store.py | 8 +- tests/sqlalchemy/test_patch.py | 4 +- tests/sqlalchemy/test_polyfill.py | 4 +- tests/testing/test_cratedb_sqlalchemy.py | 4 +- 24 files changed, 261 insertions(+), 57 deletions(-) diff --git a/.github/workflows/influxdb.yml b/.github/workflows/influxdb.yml index 50c86942..ce67c852 100644 --- a/.github/workflows/influxdb.yml +++ b/.github/workflows/influxdb.yml @@ -68,7 +68,7 @@ jobs: cache: 'pip' cache-dependency-path: 'pyproject.toml' - - name: Setup project + - name: Set up project run: | # `setuptools 0.64.0` adds support for editable install hooks (PEP 660). diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2d5867e..bffcae5b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,8 @@ concurrency: jobs: - tests: + + tests-main: runs-on: ${{ matrix.os }} strategy: @@ -43,7 +44,10 @@ jobs: - 4200:4200 - 5432:5432 - name: Python ${{ matrix.python-version }} on OS ${{ matrix.os }} + name: " + Generic: + Python ${{ matrix.python-version }} on OS ${{ matrix.os }} + " steps: - name: Acquire sources @@ -57,7 +61,7 @@ jobs: cache: 'pip' cache-dependency-path: 'pyproject.toml' - - name: Setup project + - name: Set up project run: | # `setuptools 0.64.0` adds support for editable install hooks (PEP 660). @@ -84,3 +88,66 @@ jobs: env_vars: OS,PYTHON name: codecov-umbrella fail_ci_if_error: true + + + tests-pymongo: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.9", "3.12"] + + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} + # Do not tear down Testcontainers + TC_KEEPALIVE: true + + # https://docs.github.com/en/actions/using-containerized-services/about-service-containers + services: + cratedb: + image: crate/crate:nightly + ports: + - 4200:4200 + - 5432:5432 + + name: " + PyMongo: + Python ${{ matrix.python-version }} on OS ${{ matrix.os }}" + steps: + + - name: Acquire sources + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + cache: 'pip' + cache-dependency-path: 'pyproject.toml' + + - name: Set up project + run: | + + # `setuptools 0.64.0` adds support for editable install hooks (PEP 660). + # https://github.com/pypa/setuptools/blob/main/CHANGES.rst#v6400 + pip install "setuptools>=64" --upgrade + + # Install package in editable mode. + pip install --use-pep517 --prefer-binary --editable=.[pymongo,test,develop] + + - name: Run linter and software tests + run: | + poe check + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + files: ./coverage.xml + flags: pymongo + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false diff --git a/.github/workflows/mongodb.yml b/.github/workflows/mongodb.yml index 3592aaa6..734fd991 100644 --- a/.github/workflows/mongodb.yml +++ b/.github/workflows/mongodb.yml @@ -62,7 +62,7 @@ jobs: cache: 'pip' cache-dependency-path: 'pyproject.toml' - - name: Setup project + - name: Set up project run: | # `setuptools 0.64.0` adds support for editable install hooks (PEP 660). diff --git a/cratedb_toolkit/adapter/pymongo/README.md b/cratedb_toolkit/adapter/pymongo/README.md index d13134d3..b3aaca54 100644 --- a/cratedb_toolkit/adapter/pymongo/README.md +++ b/cratedb_toolkit/adapter/pymongo/README.md @@ -27,7 +27,7 @@ with PyMongoCrateDbAdapter(dburi="crate://crate@localhost:4200"): inserted_id = collection.insert_one({"author": "Mike", "text": "My first blog post!"}).inserted_id print(inserted_id) - document = collection.find({"author": "Mike"}) + document = collection.find_one({"author": "Mike"}) print(document) ``` diff --git a/cratedb_toolkit/adapter/pymongo/backlog.md b/cratedb_toolkit/adapter/pymongo/backlog.md index 58ef0ab4..5c80b059 100644 --- a/cratedb_toolkit/adapter/pymongo/backlog.md +++ b/cratedb_toolkit/adapter/pymongo/backlog.md @@ -6,6 +6,8 @@ - Add documentation. - Add missing essential querying features: Examples: sort order, skip, limit - Add missing essential methods. Example: `db.my_collection.drop()`. +- Make write-synchronization behavior (refresh table) configurable. +- Handle deeply nested documents of various types. ## Iteration +2 @@ -24,6 +26,7 @@ ```python jessiql.exc.InvalidColumnError: Invalid column "x" for "Surrogate" specified in sort ``` +- Run (parts of) the PyMongo test suite? ## Done diff --git a/cratedb_toolkit/adapter/pymongo/collection.py b/cratedb_toolkit/adapter/pymongo/collection.py index 4cd30d1b..a430e7f4 100644 --- a/cratedb_toolkit/adapter/pymongo/collection.py +++ b/cratedb_toolkit/adapter/pymongo/collection.py @@ -1,3 +1,6 @@ +# Make Python 3.7 and 3.8 support generic types like `dict` instead of `typing.Dict`. +from __future__ import annotations + import io import logging from collections import abc diff --git a/cratedb_toolkit/adapter/pymongo/cursor.py b/cratedb_toolkit/adapter/pymongo/cursor.py index a5695738..098a1748 100644 --- a/cratedb_toolkit/adapter/pymongo/cursor.py +++ b/cratedb_toolkit/adapter/pymongo/cursor.py @@ -1,3 +1,6 @@ +# Make Python 3.7 and 3.8 support generic types like `dict` instead of `typing.Dict`. +from __future__ import annotations + import copy import logging import warnings diff --git a/cratedb_toolkit/retention/store.py b/cratedb_toolkit/retention/store.py index 3d69679a..9a053390 100644 --- a/cratedb_toolkit/retention/store.py +++ b/cratedb_toolkit/retention/store.py @@ -9,7 +9,6 @@ from sqlalchemy import MetaData, Table from sqlalchemy.exc import ProgrammingError from sqlalchemy.orm import Session -from sqlalchemy.sql.selectable import NamedFromClause from cratedb_toolkit.retention.model import JobSettings, RetentionPolicy, RetentionStrategy from cratedb_toolkit.util.database import DatabaseAdapter, sa_is_empty @@ -36,12 +35,14 @@ def get_tags_constraints(self, tags: t.Union[t.List[str], t.Set[str]]): """ Return list of SQL WHERE constraint clauses from given tags. """ + from sqlalchemy.sql.selectable import NamedFromClause # type: ignore[attr-defined] + table: NamedFromClause = self.table # type: ignore[attr-defined] constraints = [] for tag in tags: if not tag: continue - constraint = table.c[self.tag_column][tag] != sa.Null() + constraint = table.c[self.tag_column][tag] != sa.Null() # type: ignore[attr-defined] constraints.append(constraint) return sa.and_(sa.true(), *constraints) diff --git a/examples/pymongo_adapter.py b/examples/pymongo_adapter.py index fdc18c5e..fe258938 100644 --- a/examples/pymongo_adapter.py +++ b/examples/pymongo_adapter.py @@ -24,7 +24,9 @@ ========== - https://github.com/mongodb/mongo-python-driver """ +import datetime as dt import logging +import time import pymongo from pymongo.database import Database @@ -35,7 +37,7 @@ logger = logging.getLogger(__name__) -def main(): +def mongodb_workload(): client = pymongo.MongoClient( "localhost", 27017, timeoutMS=100, connectTimeoutMS=100, socketTimeoutMS=100, serverSelectionTimeoutMS=100 ) @@ -46,45 +48,70 @@ def main(): logger.info(f"Using database: {db.name}") logger.info(f"Using collection: {db.my_collection}") - # Insert records. - inserted_id = db.my_collection.insert_one({"x": 5}).inserted_id - logger.info(f"Inserted object: {inserted_id!r}") + # TODO: Dropping a collection is not implemented yet. + # db.my_collection.drop() # noqa: ERA001 + + # Insert document. + documents = [ + { + "author": "Mike", + "text": "My first blog post!", + "tags": ["mongodb", "python", "pymongo"], + "date": dt.datetime.now(tz=dt.timezone.utc), + }, + { + "author": "Eliot", + "title": "MongoDB is fun", + "text": "and pretty easy too!", + "date": dt.datetime(2009, 11, 10, 10, 45), + }, + ] + result = db.my_collection.insert_many(documents) + logger.info(f"Inserted document identifiers: {result.inserted_ids!r}") # FIXME: Refresh table. + time.sleep(1) - # Query records. + # Query documents. document_count = db.my_collection.count_documents({}) logger.info(f"Total document count: {document_count}") # Find single document. - document = db.my_collection.find_one() + document = db.my_collection.find_one({"author": "Mike"}) logger.info(f"[find_one] Response document: {document}") - # Assorted basic find operations, with sorting and paging. - print("results:", end=" ") + # Run a few basic retrieval operations, with sorting and paging. + print("Whole collection") for item in db.my_collection.find(): - print(item["x"], end=", ") + print(item) print() - print("results:", end=" ") - for item in db.my_collection.find().sort("x", pymongo.ASCENDING): - print(item["x"], end=", ") + print("Sort ascending") + for item in db.my_collection.find().sort("author", pymongo.ASCENDING): + print(item) print() - print("results:", end=" ") - for item in db.my_collection.find().sort("x", pymongo.DESCENDING): - print(item["x"], end=", ") + print("Sort descending") + for item in db.my_collection.find().sort("author", pymongo.DESCENDING): + print(item) print() - results = [item["x"] for item in db.my_collection.find().limit(2).skip(1)] - print("results:", results) - print("length:", len(results)) + print("Paging") + for item in db.my_collection.find().limit(2).skip(1): + print(item) + print() -if __name__ == "__main__": +def main(dburi: str = None): + dburi = dburi or "crate://crate@localhost:4200" + # setup_logging(level=logging.DEBUG, width=42) # noqa: ERA001 setup_logging(level=logging.INFO, width=20) # Context manager use. - with PyMongoCrateDbAdapter(dburi="crate://crate@localhost:4200"): - main() + with PyMongoCrateDbAdapter(dburi=dburi): + mongodb_workload() + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index a3c2ceb0..574e1534 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,7 @@ dependencies = [ "tqdm<5", 'typing-extensions<5; python_version <= "3.7"', "vasuki==0.7.0", + "verlib2==0.2", "yarl<1.10", ] [project.optional-dependencies] @@ -147,6 +148,7 @@ io = [ "cr8", "dask[dataframe]>=2020", "pandas<3,>=1", + "sqlalchemy>=2", ] mongodb = [ "cratedb-toolkit[io]", @@ -157,8 +159,9 @@ mongodb = [ ] pymongo = [ "jessiql==1.0.0rc1", + "pandas<3,>=2", "pymongo<5,>=3.10.1", - "sqlalchemy<2.0", + "sqlalchemy<2", ] release = [ "build<2", diff --git a/tests/adapter/test_pymongo.py b/tests/adapter/test_pymongo.py index bb2278aa..883a39f2 100644 --- a/tests/adapter/test_pymongo.py +++ b/tests/adapter/test_pymongo.py @@ -1,14 +1,21 @@ +# ruff: noqa: E402 import datetime as dt import typing as t from unittest import mock -import pymongo import pytest +pymongo = pytest.importorskip("pymongo", reason="Skipping tests because pymongo is not installed") + +from cratedb_toolkit.testing.testcontainers.cratedb import CrateDBTestAdapter +from tests.conftest import check_sqlalchemy1 + +check_sqlalchemy1(allow_module_level=True) + from cratedb_toolkit.adapter.pymongo import PyMongoCrateDbAdapter from cratedb_toolkit.adapter.pymongo.util import AmendedObjectId from cratedb_toolkit.util.date import truncate_milliseconds -from tests.conftest import TESTDRIVE_DATA_SCHEMA, CrateDBFixture +from tests.conftest import TESTDRIVE_DATA_SCHEMA pytestmark = pytest.mark.mongodb @@ -62,7 +69,10 @@ def test_pymongo_metadata(pymongo_cratedb: PyMongoCrateDbAdapter, pymongo_client def test_pymongo_insert_one_single( - pymongo_cratedb: PyMongoCrateDbAdapter, pymongo_client: pymongo.MongoClient, cratedb: CrateDBFixture, sync_writes + pymongo_cratedb: PyMongoCrateDbAdapter, + pymongo_client: pymongo.MongoClient, + cratedb: CrateDBTestAdapter, + sync_writes, ): """ Verify a single basic data insert operation `insert_one` works well. @@ -85,7 +95,10 @@ def test_pymongo_insert_one_single( def test_pymongo_insert_one_multiple( - pymongo_cratedb: PyMongoCrateDbAdapter, pymongo_client: pymongo.MongoClient, cratedb: CrateDBFixture, sync_writes + pymongo_cratedb: PyMongoCrateDbAdapter, + pymongo_client: pymongo.MongoClient, + cratedb: CrateDBTestAdapter, + sync_writes, ): """ Verify the basic data insert operation `insert_one` works well, when called multiple times. @@ -112,7 +125,10 @@ def test_pymongo_insert_one_multiple( def test_pymongo_insert_many( - pymongo_cratedb: PyMongoCrateDbAdapter, pymongo_client: pymongo.MongoClient, cratedb: CrateDBFixture, sync_writes + pymongo_cratedb: PyMongoCrateDbAdapter, + pymongo_client: pymongo.MongoClient, + cratedb: CrateDBTestAdapter, + sync_writes, ): """ Verify the basic data insert operation `insert_many` works well. @@ -138,7 +154,10 @@ def test_pymongo_insert_many( def test_pymongo_count_documents( - pymongo_cratedb: PyMongoCrateDbAdapter, pymongo_client: pymongo.MongoClient, cratedb: CrateDBFixture, sync_writes + pymongo_cratedb: PyMongoCrateDbAdapter, + pymongo_client: pymongo.MongoClient, + cratedb: CrateDBTestAdapter, + sync_writes, ): """ Verify the `count_documents` operation works well. @@ -161,7 +180,10 @@ def test_pymongo_count_documents( def test_pymongo_roundtrip_document( - pymongo_cratedb: PyMongoCrateDbAdapter, pymongo_client: pymongo.MongoClient, cratedb: CrateDBFixture, sync_writes + pymongo_cratedb: PyMongoCrateDbAdapter, + pymongo_client: pymongo.MongoClient, + cratedb: CrateDBTestAdapter, + sync_writes, ): """ Verify round-tripping a documents works well. @@ -208,8 +230,20 @@ def test_pymongo_roundtrip_document( assert document_loaded == document_original +def test_example_program(cratedb: CrateDBTestAdapter): + """ + Verify that the program `examples/pymongo_adapter.py` works. + """ + from examples.pymongo_adapter import main + + main(dburi=cratedb.database.dburi) + + def test_pymongo_tutorial( - pymongo_cratedb: PyMongoCrateDbAdapter, pymongo_client: pymongo.MongoClient, cratedb: CrateDBFixture, sync_writes + pymongo_cratedb: PyMongoCrateDbAdapter, + pymongo_client: pymongo.MongoClient, + cratedb: CrateDBTestAdapter, + sync_writes, ): """ Verify the PyMongo Tutorial works well. diff --git a/tests/conftest.py b/tests/conftest.py index a1d05df8..73787d3b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,8 @@ import pytest import responses +import sqlalchemy as sa +from verlib2 import Version from cratedb_toolkit.testing.testcontainers.cratedb import CrateDBTestAdapter from cratedb_toolkit.util.common import setup_logging @@ -114,4 +116,40 @@ def cloud_cluster_mock(): ) +IS_SQLALCHEMY1 = Version(sa.__version__) < Version("2") +IS_SQLALCHEMY2 = Version(sa.__version__) >= Version("2") + + +@pytest.fixture(scope="module") +def needs_sqlalchemy1_module(): + """ + Use this for annotating pytest test case functions testing subsystems which need SQLAlchemy 1.x. + """ + check_sqlalchemy1() + + +def check_sqlalchemy1(**kwargs): + """ + Skip pytest test cases or modules testing subsystems which need SQLAlchemy 1.x. + """ + if not IS_SQLALCHEMY1: + raise pytest.skip("This feature or subsystem needs SQLAlchemy 1.x", **kwargs) + + +@pytest.fixture +def needs_sqlalchemy2(): + """ + Use this for annotating pytest test case functions testing subsystems which need SQLAlchemy 2.x. + """ + check_sqlalchemy2() + + +def check_sqlalchemy2(**kwargs): + """ + Skip pytest test cases or modules testing subsystems which need SQLAlchemy 2.x. + """ + if not IS_SQLALCHEMY2: + raise pytest.skip("This feature or subsystem needs SQLAlchemy 2.x", **kwargs) + + setup_logging() diff --git a/tests/io/influxdb/test_cli.py b/tests/io/influxdb/test_cli.py index 63399310..1fa346c4 100644 --- a/tests/io/influxdb/test_cli.py +++ b/tests/io/influxdb/test_cli.py @@ -15,7 +15,7 @@ from influxio.adapter import InfluxDbApiAdapter -def test_influxdb2_load_table(caplog, cratedb, influxdb): +def test_influxdb2_load_table(caplog, cratedb, influxdb, needs_sqlalchemy2): """ CLI test: Invoke `ctk load table` for InfluxDB. """ diff --git a/tests/io/mongodb/conftest.py b/tests/io/mongodb/conftest.py index 5ff938eb..2361000e 100644 --- a/tests/io/mongodb/conftest.py +++ b/tests/io/mongodb/conftest.py @@ -2,6 +2,8 @@ import pytest +from tests.conftest import check_sqlalchemy2 + logger = logging.getLogger(__name__) @@ -53,6 +55,7 @@ def mongodb_service(): """ Provide an MongoDB service instance to the test suite. """ + check_sqlalchemy2() db = MongoDBFixture() db.reset() yield db diff --git a/tests/io/mongodb/test_cli.py b/tests/io/mongodb/test_cli.py index 11d42133..0398000f 100644 --- a/tests/io/mongodb/test_cli.py +++ b/tests/io/mongodb/test_cli.py @@ -5,6 +5,7 @@ from pueblo.testing.dataframe import DataFrameFactory from cratedb_toolkit.cli import cli +from tests.conftest import check_sqlalchemy2 pytestmark = pytest.mark.mongodb @@ -12,6 +13,14 @@ pytest.importorskip("rich", reason="Skipping tests because rich is not installed") +@pytest.fixture(scope="module", autouse=True) +def check_prerequisites(): + """ + This subsystem needs SQLAlchemy 2.x. + """ + check_sqlalchemy2() + + def test_version(): """ CLI test: Invoke `migr8 --version`. diff --git a/tests/io/mongodb/test_integration.py b/tests/io/mongodb/test_integration.py index 0515d5fd..83b9f68c 100644 --- a/tests/io/mongodb/test_integration.py +++ b/tests/io/mongodb/test_integration.py @@ -6,15 +6,25 @@ import pytest -from tests.io.mongodb.conftest import RESET_DATABASES +from tests.conftest import check_sqlalchemy2 pytestmark = pytest.mark.mongodb pymongo = pytest.importorskip("pymongo", reason="Skipping tests because pymongo is not installed") pytest.importorskip("rich", reason="Skipping tests because rich is not installed") + +@pytest.fixture(scope="module", autouse=True) +def check_prerequisites(): + """ + This subsystem needs SQLAlchemy 2.x. + """ + check_sqlalchemy2() + + from cratedb_toolkit.io.mongodb.core import gather_collections from cratedb_toolkit.testing.testcontainers.mongodb import MongoDbContainerWithKeepalive +from tests.io.mongodb.conftest import RESET_DATABASES logger = logging.getLogger(__name__) diff --git a/tests/io/test_import.py b/tests/io/test_import.py index 9eeaa3d3..fa34566c 100644 --- a/tests/io/test_import.py +++ b/tests/io/test_import.py @@ -24,7 +24,7 @@ def test_import_csv_pandas(cratedb, dummy_csv): assert result == [(2,)] -def test_import_csv_dask(cratedb, dummy_csv): +def test_import_csv_dask(cratedb, dummy_csv, needs_sqlalchemy2): """ Invoke convenience function `import_csv_dask`, and verify database content. """ @@ -36,7 +36,7 @@ def test_import_csv_dask(cratedb, dummy_csv): assert result == [(2,)] -def test_import_csv_dask_with_progressbar(cratedb, dummy_csv): +def test_import_csv_dask_with_progressbar(cratedb, dummy_csv, needs_sqlalchemy2): """ Invoke convenience function `import_csv_dask`, and verify database content. This time, use `progress=True` to make Dask display a progress bar. diff --git a/tests/retention/conftest.py b/tests/retention/conftest.py index 8460adbe..5b3d5046 100644 --- a/tests/retention/conftest.py +++ b/tests/retention/conftest.py @@ -9,7 +9,7 @@ from cratedb_toolkit.testing.testcontainers.azurite import ExtendedAzuriteContainer from cratedb_toolkit.testing.testcontainers.minio import ExtendedMinioContainer from cratedb_toolkit.util.database import DatabaseAdapter, run_sql -from tests.conftest import TESTDRIVE_DATA_SCHEMA, TESTDRIVE_EXT_SCHEMA +from tests.conftest import TESTDRIVE_DATA_SCHEMA, TESTDRIVE_EXT_SCHEMA, check_sqlalchemy2 @pytest.fixture() @@ -26,6 +26,7 @@ def store(database, settings): Provide a client database adapter, which is connected to the test database instance. The retention policy database table schema has been established. """ + check_sqlalchemy2() setup_schema(settings=settings) rps = RetentionPolicyStore(settings=settings) yield rps diff --git a/tests/retention/test_cli.py b/tests/retention/test_cli.py index 01b7487c..f74bfb60 100644 --- a/tests/retention/test_cli.py +++ b/tests/retention/test_cli.py @@ -191,7 +191,7 @@ def test_run_delete_dryrun(caplog, store, database, raw_metrics, policies): assert database.count_records(raw_metrics) == 6 -def test_run_delete_with_tags_match(store, database, sensor_readings, policies): +def test_run_delete_with_tags_match(store, database, sensor_readings, policies, needs_sqlalchemy2): """ Verify a basic DELETE retention policy through the CLI, with using correct (matching) tags. """ @@ -214,7 +214,7 @@ def test_run_delete_with_tags_match(store, database, sensor_readings, policies): assert database.count_records(f'"{TESTDRIVE_DATA_SCHEMA}"."sensor_readings"') == 0 -def test_run_delete_with_tags_unknown(caplog, store, database, sensor_readings, policies): +def test_run_delete_with_tags_unknown(caplog, store, database, sensor_readings, policies, needs_sqlalchemy2): """ Verify a basic DELETE retention policy through the CLI, with using wrong (not matching) tags. """ @@ -266,7 +266,9 @@ def test_run_reallocate(store, database, raw_metrics, raw_metrics_reallocate_pol assert database.count_records(raw_metrics) == 6 -def test_run_snapshot_aws_s3(caplog, store, database, sensor_readings, sensor_readings_snapshot_policy, minio): +def test_run_snapshot_aws_s3( + caplog, store, database, sensor_readings, sensor_readings_snapshot_policy, minio, needs_sqlalchemy2 +): """ Verify the "SNAPSHOT" strategy using an object storage with AWS S3 API. Invokes `cratedb-retention run --strategy=snapshot`. diff --git a/tests/retention/test_examples.py b/tests/retention/test_examples.py index 95d2be8c..f6c4122c 100644 --- a/tests/retention/test_examples.py +++ b/tests/retention/test_examples.py @@ -5,7 +5,7 @@ import responses -def test_example_edit(store): +def test_example_edit(store, needs_sqlalchemy2): """ Verify that the program `examples/retention_edit.py` works. """ diff --git a/tests/retention/test_store.py b/tests/retention/test_store.py index f868ff2a..e146396b 100644 --- a/tests/retention/test_store.py +++ b/tests/retention/test_store.py @@ -84,7 +84,7 @@ def test_list_tags(store): assert tags == ["bar", "foo"] -def test_delete_by_tag(store): +def test_delete_by_tag(store, needs_sqlalchemy2): """ Verify deleting a retention policy by single tag. """ @@ -122,7 +122,7 @@ def test_delete_by_tag(store): assert len(store.retrieve()) == 1 -def test_delete_by_all_tags(store): +def test_delete_by_all_tags(store, needs_sqlalchemy2): """ Verify deleting a retention policy by multiple tags. @@ -178,7 +178,7 @@ def test_delete_unknown(caplog, store): assert "Retention policy not found with id: unknown" in caplog.messages -def test_delete_by_tag_unknown(caplog, store): +def test_delete_by_tag_unknown(caplog, store, needs_sqlalchemy2): """ Verify behavior when deleting items by unknown tag """ @@ -194,7 +194,7 @@ def test_delete_by_tag_unknown(caplog, store): assert "No retention policies found with tags: ['unknown']" in caplog.messages -def test_delete_by_all_tags_unknown(caplog, store): +def test_delete_by_all_tags_unknown(caplog, store, needs_sqlalchemy2): """ Verify behavior when deleting items by unknown tag """ diff --git a/tests/sqlalchemy/test_patch.py b/tests/sqlalchemy/test_patch.py index 3eb9ab70..d6539869 100644 --- a/tests/sqlalchemy/test_patch.py +++ b/tests/sqlalchemy/test_patch.py @@ -8,7 +8,7 @@ from tests.conftest import TESTDRIVE_DATA_SCHEMA -def test_inspector_vanilla(database): +def test_inspector_vanilla(database, needs_sqlalchemy2): """ Vanilla SQLAlchemy Inspector tests. """ @@ -28,7 +28,7 @@ def test_inspector_vanilla(database): assert indexes == [] -def test_inspector_patched(database): +def test_inspector_patched(database, needs_sqlalchemy2): """ Patched SQLAlchemy Inspector tests. diff --git a/tests/sqlalchemy/test_polyfill.py b/tests/sqlalchemy/test_polyfill.py index 74ae6ef7..50c57f90 100644 --- a/tests/sqlalchemy/test_polyfill.py +++ b/tests/sqlalchemy/test_polyfill.py @@ -62,7 +62,7 @@ class FooBarComposite(Base): return FooBarComposite -def test_autoincrement_vanilla(database): +def test_autoincrement_vanilla(database, needs_sqlalchemy2): """ When using a model including an autoincrement column, and not assigning a value, CrateDB will fail. """ @@ -77,7 +77,7 @@ def test_autoincrement_vanilla(database): ) -def test_autoincrement_polyfill(database): +def test_autoincrement_polyfill(database, needs_sqlalchemy2): """ When using a model including an autoincrement column, and the corresponding polyfill is installed, the procedure will succeed. diff --git a/tests/testing/test_cratedb_sqlalchemy.py b/tests/testing/test_cratedb_sqlalchemy.py index 81792cb8..46db0d32 100644 --- a/tests/testing/test_cratedb_sqlalchemy.py +++ b/tests/testing/test_cratedb_sqlalchemy.py @@ -5,7 +5,7 @@ from tests.conftest import TESTDRIVE_DATA_SCHEMA -def test_cratedb_summits(cratedb): +def test_cratedb_summits(cratedb, needs_sqlalchemy2): """ Just to verify communication with CrateDB works. """ @@ -22,7 +22,7 @@ def test_cratedb_summits(cratedb): ] -def test_database_insert(cratedb): +def test_database_insert(cratedb, needs_sqlalchemy2): """ Verify that inserting two records and reading them back works well. """