From 60aa084886d9b960d3367a851171f63526590671 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 7 Nov 2024 10:02:46 +1100 Subject: [PATCH 01/12] feat: Add a ref package --- .github/workflows/ci.yaml | 1 + .gitignore | 3 ++ Makefile | 9 +++++- packages/ref-core/pyproject.toml | 4 +++ packages/ref/README.md | 10 ++++++ packages/ref/pyproject.toml | 33 ++++++++++++++++++++ packages/ref/src/ref/__init__.py | 7 +++++ packages/ref/src/ref/py.typed | 0 packages/ref/tests/unit/test_version.py | 6 ++++ pyproject.toml | 5 +++ uv.lock | 41 +++++++++++++++++++++++++ 11 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 packages/ref/README.md create mode 100644 packages/ref/pyproject.toml create mode 100644 packages/ref/src/ref/__init__.py create mode 100644 packages/ref/src/ref/py.typed create mode 100644 packages/ref/tests/unit/test_version.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 067426f..0e85f67 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,6 +74,7 @@ jobs: make fetch-test-data - name: Run tests run: | + uv run --package ref pytest packages/ref -r a -v --doctest-modules --cov=packages/ref/src --cov-report=term uv run --package ref-core pytest packages/ref-core -r a -v --doctest-modules --cov=packages/ref-core/src --cov-report=term uv run --package ref-metrics-example pytest packages/ref-metrics-example -r a -v --doctest-modules --cov=packages/ref-metrics-example/src --cov-report=term --cov-append uv run coverage xml diff --git a/.gitignore b/.gitignore index f1f0ef4..459fc76 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,6 @@ dmypy.json # Esgpull .esgpull + +# Generated output +out diff --git a/Makefile b/Makefile index b3e9a05..0a1c023 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ pre-commit: ## run all the linting checks of the codebase .PHONY: mypy mypy: ## run mypy on the codebase MYPYPATH=stubs uv run --package ref-core mypy packages/ref-core + MYPYPATH=stubs uv run --package ref mypy packages/ref MYPYPATH=stubs uv run --package ref-metrics-example mypy packages/ref-metrics-example .PHONY: ruff-fixes @@ -37,6 +38,12 @@ ruff-fixes: ## fix the code using ruff uv run ruff check --fix uv run ruff format +.PHONY: test-ref +test-ref: ## run the tests + uv run --package ref \ + pytest packages/ref \ + -r a -v --doctest-modules --cov=packages/ref/src + .PHONY: test-core test-core: ## run the tests uv run --package ref-core \ @@ -56,7 +63,7 @@ test-integration: ## run the integration tests -r a -v .PHONY: test -test: test-core test-metrics-example test-integration ## run the tests +test: test-core test-ref test-metrics-example test-integration ## run the tests # Note on code coverage and testing: # If you want to debug what is going on with coverage, we have found diff --git a/packages/ref-core/pyproject.toml b/packages/ref-core/pyproject.toml index 15592f5..a98b09e 100644 --- a/packages/ref-core/pyproject.toml +++ b/packages/ref-core/pyproject.toml @@ -21,12 +21,16 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dependencies = [ + 'ref-core' ] [tool.uv] dev-dependencies = [ ] +[tool.uv.sources] +ref-core = { workspace = true } + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/packages/ref/README.md b/packages/ref/README.md new file mode 100644 index 0000000..2c0982f --- /dev/null +++ b/packages/ref/README.md @@ -0,0 +1,10 @@ +# ref + +The `ref` package orchestrates the tracking and execution of model benchmarking metrics +against CMIP data. + + +## Usage + +The `ref` package exposes a command line interface (CLI) that can be used to +interact with the diff --git a/packages/ref/pyproject.toml b/packages/ref/pyproject.toml new file mode 100644 index 0000000..b48c326 --- /dev/null +++ b/packages/ref/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "ref" +version = "0.1.0" +description = "Core library for the CMIP Rapid Evaluation Framework" +readme = "README.md" +authors = [ + { name = "Jared Lewis", email = "jared.lewis@climate-resource.com" } +] +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Intended Audience :: Science/Research", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering", +] +dependencies = [ + "typer>=0.12.5", +] + +[tool.uv] +dev-dependencies = [ +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/packages/ref/src/ref/__init__.py b/packages/ref/src/ref/__init__.py new file mode 100644 index 0000000..9efc516 --- /dev/null +++ b/packages/ref/src/ref/__init__.py @@ -0,0 +1,7 @@ +""" +Rapid evaluating CMIP data +""" + +import importlib.metadata + +__version__ = importlib.metadata.version("ref") diff --git a/packages/ref/src/ref/py.typed b/packages/ref/src/ref/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/packages/ref/tests/unit/test_version.py b/packages/ref/tests/unit/test_version.py new file mode 100644 index 0000000..1420eeb --- /dev/null +++ b/packages/ref/tests/unit/test_version.py @@ -0,0 +1,6 @@ +from ref import __version__ as version + + +# Placeholder to get CI working +def test_version(): + assert version == "0.1.0" diff --git a/pyproject.toml b/pyproject.toml index 511c7c6..ae5b0e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ authors = [ ] requires-python = ">=3.10" dependencies = [ + "ref", "ref-core", "ref-metrics-example", ] @@ -48,6 +49,7 @@ dev-dependencies = [ members = ["packages/*"] [tool.uv.sources] +ref = { workspace = true } ref-core = { workspace = true } ref-metrics-example = { workspace = true } @@ -93,6 +95,9 @@ formats = "ipynb,py:percent" addopts = [ "--import-mode=importlib", ] +filterwarnings = [ + 'ignore:Deprecated call to `pkg_resources.:DeprecationWarning', +] # We currently check for GPL licensed code, but this restriction may be removed diff --git a/uv.lock b/uv.lock index 7761374..5a99ebb 100644 --- a/uv.lock +++ b/uv.lock @@ -4,6 +4,7 @@ requires-python = ">=3.10" [manifest] members = [ "cmip-ref", + "ref", "ref-core", "ref-metrics-example", ] @@ -363,6 +364,7 @@ name = "cmip-ref" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "ref" }, { name = "ref-core" }, { name = "ref-metrics-example" }, ] @@ -393,6 +395,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "ref", editable = "packages/ref" }, { name = "ref-core", editable = "packages/ref-core" }, { name = "ref-metrics-example", editable = "packages/ref-metrics-example" }, ] @@ -2090,11 +2093,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/d2/3b2ab40f455a256cb6672186bea95cd97b459ce4594050132d71e76f0d6f/pyzmq-26.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:90412f2db8c02a3864cbfc67db0e3dcdbda336acf1c469526d3e869394fe001c", size = 550762 }, ] +[[package]] +name = "ref" +version = "0.1.0" +source = { editable = "packages/ref" } +dependencies = [ + { name = "typer" }, +] + +[package.metadata] +requires-dist = [{ name = "typer", specifier = ">=0.12.5" }] + [[package]] name = "ref-core" version = "0.1.0" source = { editable = "packages/ref-core" } +[package.metadata] +requires-dist = [{ name = "ref-core", editable = "packages/ref-core" }] + [[package]] name = "ref-metrics-example" version = "0.1.0" @@ -2362,6 +2379,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl", hash = "sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2", size = 1248506 }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 }, +] + [[package]] name = "six" version = "1.16.0" @@ -2634,6 +2660,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359 }, ] +[[package]] +name = "typer" +version = "0.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/58/a79003b91ac2c6890fc5d90145c662fd5771c6f11447f116b63300436bc9/typer-0.12.5.tar.gz", hash = "sha256:f592f089bedcc8ec1b974125d64851029c3b1af145f04aca64d69410f0c9b722", size = 98953 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/2b/886d13e742e514f704c33c4caa7df0f3b89e5a25ef8db02aa9ca3d9535d5/typer-0.12.5-py3-none-any.whl", hash = "sha256:62fe4e471711b147e3365034133904df3e235698399bc4de2b36c8579298d52b", size = 47288 }, +] + [[package]] name = "types-python-dateutil" version = "2.9.0.20241003" From 0720e012ab4633398d2dc6687c0261128528c2c3 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 7 Nov 2024 10:41:45 +1100 Subject: [PATCH 02/12] feat: Add a config subcommands --- packages/ref/pyproject.toml | 3 +++ packages/ref/src/ref/__init__.py | 1 + packages/ref/src/ref/cli/__init__.py | 10 ++++++++++ packages/ref/src/ref/cli/config.py | 23 +++++++++++++++++++++++ packages/ref/src/ref/cli/sync.py | 11 +++++++++++ 5 files changed, 48 insertions(+) create mode 100644 packages/ref/src/ref/cli/__init__.py create mode 100644 packages/ref/src/ref/cli/config.py create mode 100644 packages/ref/src/ref/cli/sync.py diff --git a/packages/ref/pyproject.toml b/packages/ref/pyproject.toml index b48c326..7b0ad4e 100644 --- a/packages/ref/pyproject.toml +++ b/packages/ref/pyproject.toml @@ -24,6 +24,9 @@ dependencies = [ "typer>=0.12.5", ] +[project.scripts] +ref = "ref.cli:app" + [tool.uv] dev-dependencies = [ ] diff --git a/packages/ref/src/ref/__init__.py b/packages/ref/src/ref/__init__.py index 9efc516..69dab81 100644 --- a/packages/ref/src/ref/__init__.py +++ b/packages/ref/src/ref/__init__.py @@ -5,3 +5,4 @@ import importlib.metadata __version__ = importlib.metadata.version("ref") +__core_version__ = importlib.metadata.version("ref_core") diff --git a/packages/ref/src/ref/cli/__init__.py b/packages/ref/src/ref/cli/__init__.py new file mode 100644 index 0000000..dfc84b9 --- /dev/null +++ b/packages/ref/src/ref/cli/__init__.py @@ -0,0 +1,10 @@ +"""Entrypoint for the CLI""" + +import typer + +from ref.cli import config, sync + +app = typer.Typer() + +app.command(name="sync")(sync.sync) +app.add_typer(config.app, name="config") diff --git a/packages/ref/src/ref/cli/config.py b/packages/ref/src/ref/cli/config.py new file mode 100644 index 0000000..fa95e71 --- /dev/null +++ b/packages/ref/src/ref/cli/config.py @@ -0,0 +1,23 @@ +""" +View and update the REF configuration +""" + +import typer + +app = typer.Typer(help=__doc__) + + +@app.command() +def list(): + """ + Print the current ref configuration + """ + print("config") + + +@app.command() +def update(): + """ + Print the current ref configuration + """ + print("config") diff --git a/packages/ref/src/ref/cli/sync.py b/packages/ref/src/ref/cli/sync.py new file mode 100644 index 0000000..ad4f34f --- /dev/null +++ b/packages/ref/src/ref/cli/sync.py @@ -0,0 +1,11 @@ +import typer + +app = typer.Typer() + + +@app.command() +def sync(): + """ + Placeholder command for syncing data + """ # noqa: D401 + print("syncing data") From 360e45df50a39f2b465d8b19e1834820b455d96b Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 7 Nov 2024 21:10:32 +1100 Subject: [PATCH 03/12] test: Add tests for the ref package --- packages/ref/pyproject.toml | 4 + packages/ref/src/ref/cli/__init__.py | 2 +- packages/ref/src/ref/cli/config.py | 24 ++- packages/ref/src/ref/cli/sync.py | 2 +- packages/ref/src/ref/config.py | 218 +++++++++++++++++++++ packages/ref/src/ref/constants.py | 8 + packages/ref/src/ref/env.py | 35 ++++ packages/ref/tests/conftest.py | 13 ++ packages/ref/tests/unit/cli/test_config.py | 60 ++++++ packages/ref/tests/unit/test_config.py | 97 +++++++++ pyproject.toml | 1 + stubs/environs/__init__.pyi | 12 ++ uv.lock | 46 ++++- 13 files changed, 516 insertions(+), 6 deletions(-) create mode 100644 packages/ref/src/ref/config.py create mode 100644 packages/ref/src/ref/constants.py create mode 100644 packages/ref/src/ref/env.py create mode 100644 packages/ref/tests/conftest.py create mode 100644 packages/ref/tests/unit/cli/test_config.py create mode 100644 packages/ref/tests/unit/test_config.py create mode 100644 stubs/environs/__init__.pyi diff --git a/packages/ref/pyproject.toml b/packages/ref/pyproject.toml index 7b0ad4e..82b51aa 100644 --- a/packages/ref/pyproject.toml +++ b/packages/ref/pyproject.toml @@ -21,6 +21,10 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dependencies = [ + "attrs>=24.2.0", + "cattrs>=24.1.2", + "environs>=11.0.0", + "tomlkit>=0.13.2", "typer>=0.12.5", ] diff --git a/packages/ref/src/ref/cli/__init__.py b/packages/ref/src/ref/cli/__init__.py index dfc84b9..95e7c4d 100644 --- a/packages/ref/src/ref/cli/__init__.py +++ b/packages/ref/src/ref/cli/__init__.py @@ -4,7 +4,7 @@ from ref.cli import config, sync -app = typer.Typer() +app = typer.Typer(name="ref") app.command(name="sync")(sync.sync) app.add_typer(config.app, name="config") diff --git a/packages/ref/src/ref/cli/config.py b/packages/ref/src/ref/cli/config.py index fa95e71..f729a74 100644 --- a/packages/ref/src/ref/cli/config.py +++ b/packages/ref/src/ref/cli/config.py @@ -2,21 +2,39 @@ View and update the REF configuration """ +from pathlib import Path + import typer +from ref.config import Config +from ref.constants import config_filename + app = typer.Typer(help=__doc__) @app.command() -def list(): +def list(configuration_directory: Path | None = typer.Option(None, help="Configuration directory")) -> None: """ Print the current ref configuration + + If a configuration directory is provided, + the configuration will attempted to be loaded from the specified directory. + If the configuration file is missing then a """ - print("config") + try: + if configuration_directory: + config = Config.load(configuration_directory / config_filename, allow_missing=False) + else: + config = Config.default() + except FileNotFoundError: + typer.secho("Configuration file not found", fg=typer.colors.RED) + raise typer.Exit(1) + + print(config.dumps(defaults=True)) @app.command() -def update(): +def update() -> None: """ Print the current ref configuration """ diff --git a/packages/ref/src/ref/cli/sync.py b/packages/ref/src/ref/cli/sync.py index ad4f34f..ce7b2f1 100644 --- a/packages/ref/src/ref/cli/sync.py +++ b/packages/ref/src/ref/cli/sync.py @@ -4,7 +4,7 @@ @app.command() -def sync(): +def sync() -> None: """ Placeholder command for syncing data """ # noqa: D401 diff --git a/packages/ref/src/ref/config.py b/packages/ref/src/ref/config.py new file mode 100644 index 0000000..6b3c85c --- /dev/null +++ b/packages/ref/src/ref/config.py @@ -0,0 +1,218 @@ +"""Configuration management""" + +# The basics of the configuration management takes a lot of inspiration from the +# `esgpull` configuration management system with some of the extra complexity removed. +# https://github.com/ESGF/esgf-download/blob/main/esgpull/config.py + +from pathlib import Path +from typing import Any + +import tomlkit +from attrs import Factory, define, field +from cattrs import Converter +from cattrs.gen import make_dict_unstructure_fn, override +from tomlkit import TOMLDocument + +from ref.constants import config_filename +from ref.env import env + + +def _pop_empty(d: dict[str, Any]) -> None: + keys = list(d.keys()) + for key in keys: + value = d[key] + if isinstance(value, dict): + _pop_empty(value) + if not value: + d.pop(key) + + +@define +class Paths: + """ + Common paths used by the REF application + """ + + data: Path = field(converter=Path) + db: Path = field(converter=Path) + log: Path = field(converter=Path) + tmp: Path = field(converter=Path) + + @data.default + def _data_factory(self) -> Path: + return env.path("REF_CONFIGURATION") / "data" + + @db.default + def _db_factory(self) -> Path: + return env.path("REF_CONFIGURATION") / "db" + + @log.default + def _log_factory(self) -> Path: + return env.path("REF_CONFIGURATION") / "log" + + @tmp.default + def _tmp_factory(self) -> Path: + return env.path("REF_CONFIGURATION") / "tmp" + + +@define +class Db: + """ + Database configuration + """ + + filename: str = "sqlite://ref.db" + + +@define +class Config: + """ + REF configuration + + This class is used to store the configuration of the REF application. + """ + + paths: Paths = Factory(Paths) + db: Db = Factory(Db) + _raw: TOMLDocument | None = field(init=False, default=None) + _config_file: Path | None = field(init=False, default=None) + + @classmethod + def load(cls, config_file: Path, allow_missing: bool = True) -> "Config": + """ + Load the configuration from a file + + Parameters + ---------- + config_file + Path to the configuration file. + This should be a TOML file. + + Returns + ------- + : + The configuration loaded from the file + """ + if config_file.is_file(): + with config_file.open() as fh: + doc = tomlkit.load(fh) + raw = doc + else: + if not allow_missing: + raise FileNotFoundError(f"Configuration file not found: {config_file}") + + doc = TOMLDocument() + raw = None + config = _converter_defaults.structure(doc, cls) + config._raw = raw + config._config_file = config_file + return config + + def save(self, config_file: Path | None = None) -> None: + """ + Save the configuration as a TOML file + + The configuration will be saved to the specified file. + If no file is specified, the configuration will be saved to the file + that was used to load the configuration. + + Parameters + ---------- + config_file + The file to save the configuration to + + Raises + ------ + ValueError + If no configuration file is specified and the configuration was not loaded from a file + """ + if config_file is None: + if self._config_file is None: # pragma: no cover + # I'm not sure if this is possible + raise ValueError("No configuration file specified") + config_file = self._config_file + + config_file.parent.mkdir(parents=True, exist_ok=True) + + with open(config_file, "w") as fh: + fh.write(self.dumps()) + + @classmethod + def default(cls) -> "Config": + """ + Load the default configuration + + This will load the configuration from the default configuration location, + which is typically the user's configuration directory. + This location can be overridden by setting the `REF_CONFIGURATION` environment variable. + + Returns + ------- + : + The default configuration + """ + root = env.path("REF_CONFIGURATION") + return cls.load(root / config_filename) + + def dumps(self, defaults: bool = True) -> str: + """ + Dump the configuration to a TOML string + + Parameters + ---------- + defaults + If True, include default values in the output + + Returns + ------- + : + The configuration as a TOML string + """ + return self.dump(defaults).as_string() + + def dump( + self, + defaults: bool = True, + ) -> TOMLDocument: + """ + Dump the configuration to a TOML document + + Parameters + ---------- + defaults + If True, include default values in the output + + Returns + ------- + : + The configuration as a TOML document + """ + if defaults: + converter = _converter_defaults + else: + converter = _converter_no_defaults + dump = converter.unstructure(self) + if not defaults: + _pop_empty(dump) + doc = TOMLDocument() + doc.update(dump) + return doc + + +def _make_converter(omit_default: bool) -> Converter: + conv = Converter(omit_if_default=omit_default, forbid_extra_keys=True) + conv.register_unstructure_hook(Path, str) + conv.register_unstructure_hook( + Config, + make_dict_unstructure_fn( + Config, + conv, + _raw=override(omit=True), + _config_file=override(omit=True), + ), + ) + return conv + + +_converter_defaults = _make_converter(omit_default=False) +_converter_no_defaults = _make_converter(omit_default=True) diff --git a/packages/ref/src/ref/constants.py b/packages/ref/src/ref/constants.py new file mode 100644 index 0000000..1eb2211 --- /dev/null +++ b/packages/ref/src/ref/constants.py @@ -0,0 +1,8 @@ +""" +Constants used by the REF +""" + +config_filename = "ref.toml" +""" +Default name of the configuration file +""" diff --git a/packages/ref/src/ref/env.py b/packages/ref/src/ref/env.py new file mode 100644 index 0000000..4cb9782 --- /dev/null +++ b/packages/ref/src/ref/env.py @@ -0,0 +1,35 @@ +"""Environment variable management""" + +import os + +import platformdirs +from environs import Env + + +def _set_defaults() -> None: + os.environ.setdefault("REF_CONFIGURATION", str(platformdirs.user_config_path("cmip-ref"))) + + +def get_env() -> Env: + """ + Get the current environment + + Returns + ------- + : + The current environment including any environment variables loaded from the .env file + and any defaults set by this application. + """ + # Set the default values for the environment variables + _set_defaults() + + env = Env(expand_vars=True) + + # Load the environment variables from the .env file + # This will override any defaults set above + env.read_env(verbose=True) + + return env + + +env = get_env() diff --git a/packages/ref/tests/conftest.py b/packages/ref/tests/conftest.py new file mode 100644 index 0000000..9673afb --- /dev/null +++ b/packages/ref/tests/conftest.py @@ -0,0 +1,13 @@ +import pytest +from ref.config import Config + + +@pytest.fixture +def config(tmp_path, monkeypatch) -> Config: + monkeypatch.setenv("REF_CONFIGURATION", str(tmp_path / "ref")) + + # Uses the default configuration + cfg = Config.load(tmp_path / "ref" / "ref.toml") + cfg.save() + + return cfg diff --git a/packages/ref/tests/unit/cli/test_config.py b/packages/ref/tests/unit/cli/test_config.py new file mode 100644 index 0000000..29311f0 --- /dev/null +++ b/packages/ref/tests/unit/cli/test_config.py @@ -0,0 +1,60 @@ +import platformdirs +from ref.cli import app +from typer.testing import CliRunner + +runner = CliRunner() + + +def test_without_subcommand(): + result = runner.invoke(app, ["config"]) + assert result.exit_code == 2 + assert "Missing command." in result.output + + +def test_config_help(): + result = runner.invoke(app, ["config", "--help"]) + assert result.exit_code == 0 + + expected = """ + Usage: ref config [OPTIONS] COMMAND [ARGS]... + + View and update the REF configuration + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────╮ +│ list Print the current ref configuration │ +│ update Print the current ref configuration │ +╰──────────────────────────────────────────────────────────────────────────────╯ +""" + assert expected in result.output + + +class TestConfigList: + def test_config_list(self): + result = runner.invoke(app, ["config", "list"]) + assert result.exit_code == 0 + + config_dir = platformdirs.user_config_dir("cmip-ref") + assert f'data = "{config_dir}/data"\n' in result.output + assert 'filename = "sqlite://ref.db"\n' in result.output + + def test_config_list_custom(self, config): + result = runner.invoke( + app, ["config", "list", "--configuration-directory", str(config._config_file.parent)] + ) + assert result.exit_code == 0 + + def test_config_list_custom_missing(self, config): + result = runner.invoke(app, ["config", "list", "--configuration-directory", "missing"]) + assert result.exit_code == 1 + + +class TestConfigUpdate: + def test_config_update(self): + result = runner.invoke(app, ["config", "update"]) + assert result.exit_code == 0 + + # TODO: actually implement this functionality + assert "config" in result.output diff --git a/packages/ref/tests/unit/test_config.py b/packages/ref/tests/unit/test_config.py new file mode 100644 index 0000000..81ec7b4 --- /dev/null +++ b/packages/ref/tests/unit/test_config.py @@ -0,0 +1,97 @@ +from pathlib import Path + +import cattrs +import pytest +from ref.config import Config, Paths + + +class TestConfig: + def test_load_missing(self, tmp_path, monkeypatch): + monkeypatch.setenv("REF_CONFIGURATION", str(tmp_path / "ref")) + + # The configuration file doesn't exist + # so it should default to some sane defaults + assert not (tmp_path / "ref.toml").exists() + + loaded = Config.load(Path("ref.toml")) + + assert loaded.paths.data == tmp_path / "ref" / "data" + assert loaded.paths.db == tmp_path / "ref" / "db" + + # The results aren't serialised back to disk + assert not (tmp_path / "ref.toml").exists() + assert loaded._raw is None + assert loaded._config_file == Path("ref.toml") + + def test_default(self, config): + config.paths.data = "data" + config.save() + + # The default location is overridden in the config fixture + loaded = Config.default() + assert loaded.paths.data == Path("data") + + def test_load(self, config, tmp_path): + res = config.dump(defaults=True) + + with open(tmp_path / "ref.toml", "w") as fh: + fh.write(res.as_string()) + + loaded = Config.load(tmp_path / "ref.toml") + + assert config.dumps() == loaded.dumps() + + def test_load_extra_keys(self, tmp_path): + content = """[paths] +data = "data" +extra = "extra" + +[db] +filename = "sqlite://ref.db" +""" + + with open(tmp_path / "ref.toml", "w") as fh: + fh.write(content) + + # cattrs exceptions are a bit ugly, but you get an exception like this: + # + # | cattrs.errors.ClassValidationError: While structuring Config (1 sub-exception) + # +-+---------------- 1 ---------------- + # | Exception Group Traceback (most recent call last): + # | File "", line 6, in structure_Config + # | res['paths'] = __c_structure_paths(o['paths'], __c_type_paths) + # | File "", line 31, in structure_Paths + # | if errors: raise __c_cve('While structuring ' + 'Paths', errors, __cl) + # | cattrs.errors.ClassValidationError: While structuring Paths (1 sub-exception) + # | Structuring class Config @ attribute paths + # +-+---------------- 1 ---------------- + # | cattrs.errors.ForbiddenExtraKeysError: Extra fields in constructor for Paths: extra + + with pytest.raises(cattrs.errors.ClassValidationError): + Config.load(tmp_path / "ref.toml") + + def test_save(self, tmp_path): + config = Config(paths=Paths(data=Path("data"))) + + with pytest.raises(ValueError): + # The configuration file hasn't been set as it was created directly + config.save() + + config.save(tmp_path / "ref.toml") + + assert (tmp_path / "ref.toml").exists() + + def test_defaults(self, monkeypatch): + monkeypatch.setenv("REF_CONFIGURATION", "test") + + cfg = Config.load(Path("test.toml")) + + with_defaults = cfg.dump(defaults=True) + + without_defaults = cfg.dump(defaults=False) + + assert without_defaults == {} + assert with_defaults == { + "paths": {"data": "test/data", "db": "test/db", "log": "test/log", "tmp": "test/tmp"}, + "db": {"filename": "sqlite://ref.db"}, + } diff --git a/pyproject.toml b/pyproject.toml index ae5b0e6..dabebb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,7 @@ addopts = [ ] filterwarnings = [ 'ignore:Deprecated call to `pkg_resources.:DeprecationWarning', + 'ignore:pkg_resources is deprecated as an API:DeprecationWarning', ] diff --git a/stubs/environs/__init__.pyi b/stubs/environs/__init__.pyi new file mode 100644 index 0000000..9f4ae8c --- /dev/null +++ b/stubs/environs/__init__.pyi @@ -0,0 +1,12 @@ +from pathlib import Path + +class Env: + def __init__(self, expand_vars: bool) -> None: ... + def path(self, name: str) -> Path: ... + def read_env( + self, + path: str | None = None, + recurse: bool = True, + verbose: bool = False, + override: bool = False, + ) -> bool: ... diff --git a/uv.lock b/uv.lock index 5a99ebb..2f3f2f5 100644 --- a/uv.lock +++ b/uv.lock @@ -611,6 +611,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, ] +[[package]] +name = "environs" +version = "11.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "marshmallow" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/13/3d448cfbed9f1baff5765f49434cd849501351f14fd3f09f0f2e9bd35322/environs-11.0.0.tar.gz", hash = "sha256:069727a8f73d8ba8d033d3cd95c0da231d44f38f1da773bf076cef168d312ee8", size = 25787 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/30/ef8a3022e6cdcedfd7ba03ca88ab29e30334f8e958cdbf5ce120912397e8/environs-11.0.0-py3-none-any.whl", hash = "sha256:e0bcfd41c718c07a7db422f9109e490746450da38793fe4ee197f397b9343435", size = 12216 }, +] + [[package]] name = "esgpull" version = "0.7.3" @@ -1280,6 +1293,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/87/4c364e0f109eea2402079abecbe33fef4f347b551a11423d1f4e187ea497/MarkupSafe-3.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:730d86af59e0e43ce277bb83970530dd223bf7f2a838e086b50affa6ec5f9295", size = 15741 }, ] +[[package]] +name = "marshmallow" +version = "3.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/30/14d8609f65c8aeddddd3181c06d2c9582da6278f063b27c910bbf9903441/marshmallow-3.23.1.tar.gz", hash = "sha256:3a8dfda6edd8dcdbf216c0ede1d1e78d230a6dc9c5a088f58c4083b974a0d468", size = 177488 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/a7/a78ff54e67ef92a3d12126b98eb98ab8abab3de4a8c46d240c87e514d6bb/marshmallow-3.23.1-py3-none-any.whl", hash = "sha256:fece2eb2c941180ea1b7fcbd4a83c51bfdd50093fdd3ad2585ee5e1df2508491", size = 49488 }, +] + [[package]] name = "matplotlib-inline" version = "0.1.7" @@ -1925,6 +1950,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, ] +[[package]] +name = "python-dotenv" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/57/e84d88dfe0aec03b7a2d4327012c1627ab5f03652216c63d49846d7a6c58/python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca", size = 39115 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", size = 19863 }, +] + [[package]] name = "python-json-logger" version = "2.0.7" @@ -2098,11 +2132,21 @@ name = "ref" version = "0.1.0" source = { editable = "packages/ref" } dependencies = [ + { name = "attrs" }, + { name = "cattrs" }, + { name = "environs" }, + { name = "tomlkit" }, { name = "typer" }, ] [package.metadata] -requires-dist = [{ name = "typer", specifier = ">=0.12.5" }] +requires-dist = [ + { name = "attrs", specifier = ">=24.2.0" }, + { name = "cattrs", specifier = ">=24.1.2" }, + { name = "environs", specifier = ">=11.0.0" }, + { name = "tomlkit", specifier = ">=0.13.2" }, + { name = "typer", specifier = ">=0.12.5" }, +] [[package]] name = "ref-core" From 9ee2144f6348638a858186040aa734502a6c5d5d Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 7 Nov 2024 21:59:09 +1100 Subject: [PATCH 04/12] chore: Fix tests --- packages/ref-core/src/ref_core/exceptions.py | 4 ++++ packages/ref/src/ref/cli/config.py | 5 ++--- packages/ref/tests/unit/cli/test_config.py | 15 +-------------- 3 files changed, 7 insertions(+), 17 deletions(-) create mode 100644 packages/ref-core/src/ref_core/exceptions.py diff --git a/packages/ref-core/src/ref_core/exceptions.py b/packages/ref-core/src/ref_core/exceptions.py new file mode 100644 index 0000000..0e9eb8f --- /dev/null +++ b/packages/ref-core/src/ref_core/exceptions.py @@ -0,0 +1,4 @@ +class RefException(Exception): + """Base class for exceptions related to REF operations""" + + pass diff --git a/packages/ref/src/ref/cli/config.py b/packages/ref/src/ref/cli/config.py index f729a74..c220e20 100644 --- a/packages/ref/src/ref/cli/config.py +++ b/packages/ref/src/ref/cli/config.py @@ -18,8 +18,7 @@ def list(configuration_directory: Path | None = typer.Option(None, help="Configu Print the current ref configuration If a configuration directory is provided, - the configuration will attempted to be loaded from the specified directory. - If the configuration file is missing then a + the configuration will attempt to load from the specified directory. """ try: if configuration_directory: @@ -36,6 +35,6 @@ def list(configuration_directory: Path | None = typer.Option(None, help="Configu @app.command() def update() -> None: """ - Print the current ref configuration + Update a configuration value """ print("config") diff --git a/packages/ref/tests/unit/cli/test_config.py b/packages/ref/tests/unit/cli/test_config.py index 29311f0..8423a9f 100644 --- a/packages/ref/tests/unit/cli/test_config.py +++ b/packages/ref/tests/unit/cli/test_config.py @@ -15,20 +15,7 @@ def test_config_help(): result = runner.invoke(app, ["config", "--help"]) assert result.exit_code == 0 - expected = """ - Usage: ref config [OPTIONS] COMMAND [ARGS]... - - View and update the REF configuration - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────╮ -│ list Print the current ref configuration │ -│ update Print the current ref configuration │ -╰──────────────────────────────────────────────────────────────────────────────╯ -""" - assert expected in result.output + assert "View and update the REF configuration" in result.output class TestConfigList: From b507ad1038516556d83c631dc95d16605772badf Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 7 Nov 2024 22:12:25 +1100 Subject: [PATCH 05/12] docs: Changelog --- changelog/8.feature.md | 1 + packages/ref-core/pyproject.toml | 4 ---- packages/ref/pyproject.toml | 3 ++- uv.lock | 5 ++--- 4 files changed, 5 insertions(+), 8 deletions(-) create mode 100644 changelog/8.feature.md diff --git a/changelog/8.feature.md b/changelog/8.feature.md new file mode 100644 index 0000000..ca79582 --- /dev/null +++ b/changelog/8.feature.md @@ -0,0 +1 @@ +Adds the `ref` package with a basic CLI interface that will allow for users to interact with the database of jobs. diff --git a/packages/ref-core/pyproject.toml b/packages/ref-core/pyproject.toml index a98b09e..15592f5 100644 --- a/packages/ref-core/pyproject.toml +++ b/packages/ref-core/pyproject.toml @@ -21,16 +21,12 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dependencies = [ - 'ref-core' ] [tool.uv] dev-dependencies = [ ] -[tool.uv.sources] -ref-core = { workspace = true } - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/packages/ref/pyproject.toml b/packages/ref/pyproject.toml index 82b51aa..809c96a 100644 --- a/packages/ref/pyproject.toml +++ b/packages/ref/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "ref" version = "0.1.0" -description = "Core library for the CMIP Rapid Evaluation Framework" +description = "Application which runs the CMIP Rapid Evaluation Framework" readme = "README.md" authors = [ { name = "Jared Lewis", email = "jared.lewis@climate-resource.com" } @@ -21,6 +21,7 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dependencies = [ + "ref-core", "attrs>=24.2.0", "cattrs>=24.1.2", "environs>=11.0.0", diff --git a/uv.lock b/uv.lock index 2f3f2f5..9dc4363 100644 --- a/uv.lock +++ b/uv.lock @@ -2135,6 +2135,7 @@ dependencies = [ { name = "attrs" }, { name = "cattrs" }, { name = "environs" }, + { name = "ref-core" }, { name = "tomlkit" }, { name = "typer" }, ] @@ -2144,6 +2145,7 @@ requires-dist = [ { name = "attrs", specifier = ">=24.2.0" }, { name = "cattrs", specifier = ">=24.1.2" }, { name = "environs", specifier = ">=11.0.0" }, + { name = "ref-core", editable = "packages/ref-core" }, { name = "tomlkit", specifier = ">=0.13.2" }, { name = "typer", specifier = ">=0.12.5" }, ] @@ -2153,9 +2155,6 @@ name = "ref-core" version = "0.1.0" source = { editable = "packages/ref-core" } -[package.metadata] -requires-dist = [{ name = "ref-core", editable = "packages/ref-core" }] - [[package]] name = "ref-metrics-example" version = "0.1.0" From 263319cc3893ab8aa6f36433670d7768dadf64c9 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 8 Nov 2024 11:34:26 +1100 Subject: [PATCH 06/12] feat: Add sqlalchemy and alembic --- alembic.ini | 114 ++++++++++++++++++ packages/ref/alembic/README | 22 ++++ packages/ref/alembic/env.py | 73 +++++++++++ packages/ref/alembic/script.py.mako | 26 ++++ .../alembic/versions/0.1.0_initial_table.py | 41 +++++++ packages/ref/pyproject.toml | 3 + packages/ref/src/ref/config.py | 2 +- packages/ref/src/ref/database.py | 66 ++++++++++ packages/ref/src/ref/models/__init__.py | 10 ++ packages/ref/src/ref/models/base.py | 9 ++ packages/ref/src/ref/models/dataset.py | 46 +++++++ ruff.toml | 5 +- uv.lock | 92 +++++++++----- 13 files changed, 479 insertions(+), 30 deletions(-) create mode 100644 alembic.ini create mode 100644 packages/ref/alembic/README create mode 100644 packages/ref/alembic/env.py create mode 100644 packages/ref/alembic/script.py.mako create mode 100644 packages/ref/alembic/versions/0.1.0_initial_table.py create mode 100644 packages/ref/src/ref/database.py create mode 100644 packages/ref/src/ref/models/__init__.py create mode 100644 packages/ref/src/ref/models/base.py create mode 100644 packages/ref/src/ref/models/dataset.py diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..b22849f --- /dev/null +++ b/alembic.ini @@ -0,0 +1,114 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +# Use forward slashes (/) also on windows to provide an os agnostic path +script_location = %(here)s/packages/ref/alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to packages/ref/alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:packages/ref/alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +# version_path_separator = newline +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# lint with attempts to fix using "ruff" +hooks = ruff-fix, ruff-format + +ruff-fix.type = exec +ruff-fix.executable = %(here)s/.venv/bin/ruff +ruff-fix.options = check -q --fix REVISION_SCRIPT_FILENAME + +ruff-format.type = exec +ruff-format.executable = %(here)s/.venv/bin/ruff +ruff-format.options = format -q REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/packages/ref/alembic/README b/packages/ref/alembic/README new file mode 100644 index 0000000..f390f77 --- /dev/null +++ b/packages/ref/alembic/README @@ -0,0 +1,22 @@ +# Alembic + +Alembic is a database migration tool. +It interoperates with SqlAlchemy to determine how the currently declared models differ from what the database +expects and generates a migration to apply the changes. + +The migrations are applied at run-time automatically (see [ref.database.Database]()). + +## Generating migrations + +To generate a migration, +you can use the `uv run` command with the `alembic` package and the `revision` command. +The `--rev-id` flag is used to specify the revision id. +If it is omitted the revision id will be generated automatically. + +``` +uv run --package ref alembic revision --rev-id 0.1.0 --message "initial table" --autogenerate +``` + +How we name and manage these migrations is still a work in progress. +It might be nice to have a way to automatically generate the revision id based on the version of the package. +This would allow us to easily track which migrations have been applied to the database. diff --git a/packages/ref/alembic/env.py b/packages/ref/alembic/env.py new file mode 100644 index 0000000..9d85fe3 --- /dev/null +++ b/packages/ref/alembic/env.py @@ -0,0 +1,73 @@ +from logging.config import fileConfig + +from alembic import context + +from ref.config import Config +from ref.database import Database +from ref.models import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = config.attributes.get("connection", None) + + if connectable is None: + db = Database.from_config(Config(), run_migrations=False) + connectable = db._engine + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/packages/ref/alembic/script.py.mako b/packages/ref/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/packages/ref/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/packages/ref/alembic/versions/0.1.0_initial_table.py b/packages/ref/alembic/versions/0.1.0_initial_table.py new file mode 100644 index 0000000..2413d0d --- /dev/null +++ b/packages/ref/alembic/versions/0.1.0_initial_table.py @@ -0,0 +1,41 @@ +"""initial table + +Revision ID: 0.1.0 +Revises: +Create Date: 2024-11-08 11:21:55.995923 + +""" + +from collections.abc import Sequence +from typing import Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0.1.0" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "dataset", + sa.Column("dataset_id", sa.String(), nullable=False), + sa.Column("instance_id", sa.String(), nullable=False), + sa.Column("master_id", sa.String(), nullable=False), + sa.Column("version", sa.String(), nullable=False), + sa.Column("data_node", sa.String(), nullable=False), + sa.Column("size", sa.Integer(), nullable=False), + sa.Column("number_of_files", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("dataset_id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("dataset") + # ### end Alembic commands ### diff --git a/packages/ref/pyproject.toml b/packages/ref/pyproject.toml index 809c96a..55962ea 100644 --- a/packages/ref/pyproject.toml +++ b/packages/ref/pyproject.toml @@ -27,6 +27,9 @@ dependencies = [ "environs>=11.0.0", "tomlkit>=0.13.2", "typer>=0.12.5", + "sqlalchemy>=2.0.36", + "alembic>=1.13.3", + "loguru>=0.7.2", ] [project.scripts] diff --git a/packages/ref/src/ref/config.py b/packages/ref/src/ref/config.py index 6b3c85c..9aa8333 100644 --- a/packages/ref/src/ref/config.py +++ b/packages/ref/src/ref/config.py @@ -61,7 +61,7 @@ class Db: Database configuration """ - filename: str = "sqlite://ref.db" + filename: str = "ref.db" @define diff --git a/packages/ref/src/ref/database.py b/packages/ref/src/ref/database.py new file mode 100644 index 0000000..b8ad352 --- /dev/null +++ b/packages/ref/src/ref/database.py @@ -0,0 +1,66 @@ +from pathlib import Path + +import alembic.command +import sqlalchemy +from alembic.config import Config as AlembicConfig +from alembic.script import ScriptDirectory +from loguru import logger +from sqlalchemy.orm import Session + +from ref.config import Config + + +class Database: + """ + Manage the database connection and migrations + + The database migrations are optionally run after the connection to the database is established. + """ + + def __init__(self, url: str, run_migrations: bool = True) -> None: + logger.info(f"Connecting to database at {url}") + self.url = url + self._engine = sqlalchemy.create_engine(self.url) + self.session = Session(self._engine) + if run_migrations: + self._migrate() + + def _migrate(self): + root_dir = Path(__file__).parents[4] + + alembic_config = AlembicConfig(root_dir / "alembic.ini") + # alembic_config.set_main_option("script_location", str(root_dir / "packages" / "ref" / "alembic")) + alembic_config.attributes["connection"] = self._engine + + script = ScriptDirectory.from_config(alembic_config) + head = script.get_current_head() + + # Run migrations + alembic.command.upgrade(alembic_config, head) + + @staticmethod + def from_config(config: Config, run_migrations: bool = True) -> "Database": + """ + Create a Database instance from a Config instance + + Parameters + ---------- + config + The Config instance that includes information about where the database is located + run_migrations + If true, run the migrations when the database is loaded + + Returns + ------- + : + A new Database instance + """ + # TODO: move the database URL creation to the Config class + config.paths.db.mkdir(parents=True, exist_ok=True) + url = f"sqlite:///{config.paths.db / config.db.filename}" + + return Database(url, run_migrations=run_migrations) + + +if __name__ == "__main__": + Database.from_config(Config(), run_migrations=True) diff --git a/packages/ref/src/ref/models/__init__.py b/packages/ref/src/ref/models/__init__.py new file mode 100644 index 0000000..4e50a0b --- /dev/null +++ b/packages/ref/src/ref/models/__init__.py @@ -0,0 +1,10 @@ +""" +Declaration of the models used by the REF. + +These models are used to represent the data that is stored in the database. +""" + +from ref.models.base import Base +from ref.models.dataset import Dataset + +__all__ = ["Base", "Dataset"] diff --git a/packages/ref/src/ref/models/base.py b/packages/ref/src/ref/models/base.py new file mode 100644 index 0000000..a85b3e0 --- /dev/null +++ b/packages/ref/src/ref/models/base.py @@ -0,0 +1,9 @@ +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + """ + Base class for all models + """ + + pass diff --git a/packages/ref/src/ref/models/dataset.py b/packages/ref/src/ref/models/dataset.py new file mode 100644 index 0000000..f4bf76b --- /dev/null +++ b/packages/ref/src/ref/models/dataset.py @@ -0,0 +1,46 @@ +from sqlalchemy.orm import Mapped, mapped_column + +from ref.models.base import Base + + +class Dataset(Base): + """ + Represents a dataset + """ + + __tablename__ = "dataset" + + dataset_id: Mapped[str] = mapped_column(primary_key=True) + """ + Complete dataset identifier + + Includes the data node and version + """ + instance_id: Mapped[str] + """ + Unique identifier for the dataset + """ + master_id: Mapped[str] + """ + Identifer for the dataset (excluding version) + """ + version: Mapped[str] + """ + Version of the dataset + """ + data_node: Mapped[str] + """ + Data node where the dataset is stored + """ + size: Mapped[int] + """ + Size of the dataset in bytes + """ + number_of_files: Mapped[int] + + # Should we also track the following fields? + # variable_id, table_id, institution_id, model_id, experiment_id, source_id, member_id, grid_label, + # time_range, time_frequency, realm,retracted + + def __repr__(self): + return f"" diff --git a/ruff.toml b/ruff.toml index 3351c69..1fa58ba 100644 --- a/ruff.toml +++ b/ruff.toml @@ -48,12 +48,15 @@ ignore = [ "S101", # Use of `assert` detected "PD901", # `df` is a bad variable name. ] +"*/alembic/versions/*" = [ + "D103", # Missing docstring in public function +] "scripts/*" = [ "S101" # S101 Use of `assert` detected ] [lint.isort] -known-first-party = ["ref-core"] +known-first-party = ["ref", "ref-core"] [lint.pydocstyle] convention = "numpy" diff --git a/uv.lock b/uv.lock index 9dc4363..2a14c5e 100644 --- a/uv.lock +++ b/uv.lock @@ -1202,6 +1202,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/bb/fbc7dd6ea215b97b90c35efc8c8f3dbfcbacb91af8c806dff1f49deddd8e/liccheck-0.9.2-py2.py3-none-any.whl", hash = "sha256:15cbedd042515945fe9d58b62e0a5af2f2a7795def216f163bb35b3016a16637", size = 13652 }, ] +[[package]] +name = "loguru" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/30/d87a423766b24db416a46e9335b9602b054a72b96a88a241f2b09b560fa8/loguru-0.7.2.tar.gz", hash = "sha256:e671a53522515f34fd406340ee968cb9ecafbc4b36c679da03c18fd8d0bd51ac", size = 145103 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/0a/4f6fed21aa246c6b49b561ca55facacc2a44b87d65b8b92362a8e99ba202/loguru-0.7.2-py3-none-any.whl", hash = "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb", size = 62549 }, +] + [[package]] name = "mako" version = "1.3.6" @@ -2132,20 +2145,26 @@ name = "ref" version = "0.1.0" source = { editable = "packages/ref" } dependencies = [ + { name = "alembic" }, { name = "attrs" }, { name = "cattrs" }, { name = "environs" }, + { name = "loguru" }, { name = "ref-core" }, + { name = "sqlalchemy" }, { name = "tomlkit" }, { name = "typer" }, ] [package.metadata] requires-dist = [ + { name = "alembic", specifier = ">=1.13.3" }, { name = "attrs", specifier = ">=24.2.0" }, { name = "cattrs", specifier = ">=24.1.2" }, { name = "environs", specifier = ">=11.0.0" }, + { name = "loguru", specifier = ">=0.7.2" }, { name = "ref-core", editable = "packages/ref-core" }, + { name = "sqlalchemy", specifier = ">=2.0.36" }, { name = "tomlkit", specifier = ">=0.13.2" }, { name = "typer", specifier = ">=0.12.5" }, ] @@ -2551,39 +2570,47 @@ wheels = [ [[package]] name = "sqlalchemy" -version = "2.0.35" +version = "2.0.36" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "(python_full_version < '3.13' and platform_machine == 'AMD64') or (python_full_version < '3.13' and platform_machine == 'WIN32') or (python_full_version < '3.13' and platform_machine == 'aarch64') or (python_full_version < '3.13' and platform_machine == 'amd64') or (python_full_version < '3.13' and platform_machine == 'ppc64le') or (python_full_version < '3.13' and platform_machine == 'win32') or (python_full_version < '3.13' and platform_machine == 'x86_64')" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/48/4f190a83525f5cefefa44f6adc9e6386c4de5218d686c27eda92eb1f5424/sqlalchemy-2.0.35.tar.gz", hash = "sha256:e11d7ea4d24f0a262bccf9a7cd6284c976c5369dac21db237cff59586045ab9f", size = 9562798 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/61/19395d0ae78c94f6f80c8adf39a142f3fe56cfb2235d8f2317d6dae1bf0e/SQLAlchemy-2.0.35-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:67219632be22f14750f0d1c70e62f204ba69d28f62fd6432ba05ab295853de9b", size = 2090086 }, - { url = "https://files.pythonhosted.org/packages/e6/82/06b5fcbe5d49043e40cf4e01e3b33c471c8d9292d478420b08538cae8928/SQLAlchemy-2.0.35-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4668bd8faf7e5b71c0319407b608f278f279668f358857dbfd10ef1954ac9f90", size = 2081278 }, - { url = "https://files.pythonhosted.org/packages/68/d1/7fb7ee46949a5fb34005795b1fc06a8fef67587a66da731c14e545f7eb5b/SQLAlchemy-2.0.35-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb8bea573863762bbf45d1e13f87c2d2fd32cee2dbd50d050f83f87429c9e1ea", size = 3063763 }, - { url = "https://files.pythonhosted.org/packages/7e/ff/a1eacd78b31e52a5073e9924fb4722ecc2a72f093ca8181ed81fc61aed2e/SQLAlchemy-2.0.35-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f552023710d4b93d8fb29a91fadf97de89c5926c6bd758897875435f2a939f33", size = 3072032 }, - { url = "https://files.pythonhosted.org/packages/21/ae/ddfecf149a6d16af87408bca7bd108eef7ef23d376cc8464317efb3cea3f/SQLAlchemy-2.0.35-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:016b2e665f778f13d3c438651dd4de244214b527a275e0acf1d44c05bc6026a9", size = 3028092 }, - { url = "https://files.pythonhosted.org/packages/cc/51/3e84d42121662a160bacd311cfacb29c1e6a229d59dd8edb09caa8ab283b/SQLAlchemy-2.0.35-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7befc148de64b6060937231cbff8d01ccf0bfd75aa26383ffdf8d82b12ec04ff", size = 3053543 }, - { url = "https://files.pythonhosted.org/packages/3e/7a/039c78105958da3fc361887f0a82c974cb6fa5bba965c1689ec778be1c01/SQLAlchemy-2.0.35-cp310-cp310-win32.whl", hash = "sha256:22b83aed390e3099584b839b93f80a0f4a95ee7f48270c97c90acd40ee646f0b", size = 2062372 }, - { url = "https://files.pythonhosted.org/packages/a2/50/f31e927d32f9729f69d150ffe47e7cf51e3e0bb2148fc400b3e93a92ca4c/SQLAlchemy-2.0.35-cp310-cp310-win_amd64.whl", hash = "sha256:a29762cd3d116585278ffb2e5b8cc311fb095ea278b96feef28d0b423154858e", size = 2086485 }, - { url = "https://files.pythonhosted.org/packages/c3/46/9215a35bf98c3a2528e987791e6180eb51624d2c7d5cb8e2d96a6450b657/SQLAlchemy-2.0.35-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e21f66748ab725ade40fa7af8ec8b5019c68ab00b929f6643e1b1af461eddb60", size = 2091274 }, - { url = "https://files.pythonhosted.org/packages/1e/69/919673c5101a0c633658d58b11b454b251ca82300941fba801201434755d/SQLAlchemy-2.0.35-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8a6219108a15fc6d24de499d0d515c7235c617b2540d97116b663dade1a54d62", size = 2081672 }, - { url = "https://files.pythonhosted.org/packages/67/ea/a6b0597cbda12796be2302153369dbbe90573fdab3bc4885f8efac499247/SQLAlchemy-2.0.35-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:042622a5306c23b972192283f4e22372da3b8ddf5f7aac1cc5d9c9b222ab3ff6", size = 3200083 }, - { url = "https://files.pythonhosted.org/packages/8c/d6/97bdc8d714fb21762f2092511f380f18cdb2d985d516071fa925bb433a90/SQLAlchemy-2.0.35-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:627dee0c280eea91aed87b20a1f849e9ae2fe719d52cbf847c0e0ea34464b3f7", size = 3200080 }, - { url = "https://files.pythonhosted.org/packages/87/d2/8c2adaf2ade4f6f1b725acd0b0be9210bb6a2df41024729a8eec6a86fe5a/SQLAlchemy-2.0.35-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4fdcd72a789c1c31ed242fd8c1bcd9ea186a98ee8e5408a50e610edfef980d71", size = 3137108 }, - { url = "https://files.pythonhosted.org/packages/7e/ae/ea05d0bfa8f2b25ae34591895147152854fc950f491c4ce362ae06035db8/SQLAlchemy-2.0.35-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:89b64cd8898a3a6f642db4eb7b26d1b28a497d4022eccd7717ca066823e9fb01", size = 3157437 }, - { url = "https://files.pythonhosted.org/packages/fe/5d/8ad6df01398388a766163d27960b3365f1bbd8bb7b05b5cad321a8b69b25/SQLAlchemy-2.0.35-cp311-cp311-win32.whl", hash = "sha256:6a93c5a0dfe8d34951e8a6f499a9479ffb9258123551fa007fc708ae2ac2bc5e", size = 2061935 }, - { url = "https://files.pythonhosted.org/packages/ff/68/8557efc0c32c8e2c147cb6512237448b8ed594a57cd015fda67f8e56bb3f/SQLAlchemy-2.0.35-cp311-cp311-win_amd64.whl", hash = "sha256:c68fe3fcde03920c46697585620135b4ecfdfc1ed23e75cc2c2ae9f8502c10b8", size = 2087281 }, - { url = "https://files.pythonhosted.org/packages/2f/2b/fff87e6db0da31212c98bbc445f83fb608ea92b96bda3f3f10e373bac76c/SQLAlchemy-2.0.35-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:eb60b026d8ad0c97917cb81d3662d0b39b8ff1335e3fabb24984c6acd0c900a2", size = 2089790 }, - { url = "https://files.pythonhosted.org/packages/68/92/4bb761bd82764d5827bf6b6095168c40fb5dbbd23670203aef2f96ba6bc6/SQLAlchemy-2.0.35-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6921ee01caf375363be5e9ae70d08ce7ca9d7e0e8983183080211a062d299468", size = 2080266 }, - { url = "https://files.pythonhosted.org/packages/22/46/068a65db6dc253c6f25a7598d99e0a1d60b14f661f9d09ef6c73c718fa4e/SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cdf1a0dbe5ced887a9b127da4ffd7354e9c1a3b9bb330dce84df6b70ccb3a8d", size = 3229760 }, - { url = "https://files.pythonhosted.org/packages/6e/36/59830dafe40dda592304debd4cd86e583f63472f3a62c9e2695a5795e786/SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93a71c8601e823236ac0e5d087e4f397874a421017b3318fd92c0b14acf2b6db", size = 3240649 }, - { url = "https://files.pythonhosted.org/packages/00/50/844c50c6996f9c7f000c959dd1a7436a6c94e449ee113046a1d19e470089/SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e04b622bb8a88f10e439084486f2f6349bf4d50605ac3e445869c7ea5cf0fa8c", size = 3176138 }, - { url = "https://files.pythonhosted.org/packages/df/d2/336b18cac68eecb67de474fc15c85f13be4e615c6f5bae87ea38c6734ce0/SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1b56961e2d31389aaadf4906d453859f35302b4eb818d34a26fab72596076bb8", size = 3202753 }, - { url = "https://files.pythonhosted.org/packages/f0/f3/ee1e62fabdc10910b5ef720ae08e59bc785f26652876af3a50b89b97b412/SQLAlchemy-2.0.35-cp312-cp312-win32.whl", hash = "sha256:0f9f3f9a3763b9c4deb8c5d09c4cc52ffe49f9876af41cc1b2ad0138878453cf", size = 2060113 }, - { url = "https://files.pythonhosted.org/packages/60/63/a3cef44a52979169d884f3583d0640e64b3c28122c096474a1d7cfcaf1f3/SQLAlchemy-2.0.35-cp312-cp312-win_amd64.whl", hash = "sha256:25b0f63e7fcc2a6290cb5f7f5b4fc4047843504983a28856ce9b35d8f7de03cc", size = 2085839 }, - { url = "https://files.pythonhosted.org/packages/0e/c6/33c706449cdd92b1b6d756b247761e27d32230fd6b2de5f44c4c3e5632b2/SQLAlchemy-2.0.35-py3-none-any.whl", hash = "sha256:2ab3f0336c0387662ce6221ad30ab3a5e6499aab01b9790879b6578fd9b8faa1", size = 1881276 }, +sdist = { url = "https://files.pythonhosted.org/packages/50/65/9cbc9c4c3287bed2499e05033e207473504dc4df999ce49385fb1f8b058a/sqlalchemy-2.0.36.tar.gz", hash = "sha256:7f2767680b6d2398aea7082e45a774b2b0767b5c8d8ffb9c8b683088ea9b29c5", size = 9574485 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/72/14ab694b8b3f0e35ef5beb74a8fea2811aa791ba1611c44dc90cdf46af17/SQLAlchemy-2.0.36-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59b8f3adb3971929a3e660337f5dacc5942c2cdb760afcabb2614ffbda9f9f72", size = 2092604 }, + { url = "https://files.pythonhosted.org/packages/1e/59/333fcbca58b79f5b8b61853d6137530198823392151fa8fd9425f367519e/SQLAlchemy-2.0.36-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37350015056a553e442ff672c2d20e6f4b6d0b2495691fa239d8aa18bb3bc908", size = 2083796 }, + { url = "https://files.pythonhosted.org/packages/6c/a0/ec3c188d2b0c1bc742262e76408d44104598d7247c23f5b06bb97ee21bfa/SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8318f4776c85abc3f40ab185e388bee7a6ea99e7fa3a30686580b209eaa35c08", size = 3066165 }, + { url = "https://files.pythonhosted.org/packages/07/15/68ef91de5b8b7f80fb2d2b3b31ed42180c6227fe0a701aed9d01d34f98ec/SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c245b1fbade9c35e5bd3b64270ab49ce990369018289ecfde3f9c318411aaa07", size = 3074428 }, + { url = "https://files.pythonhosted.org/packages/e2/4c/9dfea5e63b87325eef6d9cdaac913459aa6a157a05a05ea6ff20004aee8e/SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:69f93723edbca7342624d09f6704e7126b152eaed3cdbb634cb657a54332a3c5", size = 3030477 }, + { url = "https://files.pythonhosted.org/packages/16/a5/fcfde8e74ea5f683b24add22463bfc21e431d4a5531c8a5b55bc6fbea164/SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f9511d8dd4a6e9271d07d150fb2f81874a3c8c95e11ff9af3a2dfc35fe42ee44", size = 3055942 }, + { url = "https://files.pythonhosted.org/packages/3c/ee/c22c415a771d791ae99146d72ffdb20e43625acd24835ea7fc157436d59f/SQLAlchemy-2.0.36-cp310-cp310-win32.whl", hash = "sha256:c3f3631693003d8e585d4200730616b78fafd5a01ef8b698f6967da5c605b3fa", size = 2064960 }, + { url = "https://files.pythonhosted.org/packages/aa/af/ad9c25cadc79bd851bdb9d82b68af9bdb91ff05f56d0da2f8a654825974f/SQLAlchemy-2.0.36-cp310-cp310-win_amd64.whl", hash = "sha256:a86bfab2ef46d63300c0f06936bd6e6c0105faa11d509083ba8f2f9d237fb5b5", size = 2089078 }, + { url = "https://files.pythonhosted.org/packages/00/4e/5a67963fd7cbc1beb8bd2152e907419f4c940ef04600b10151a751fe9e06/SQLAlchemy-2.0.36-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fd3a55deef00f689ce931d4d1b23fa9f04c880a48ee97af488fd215cf24e2a6c", size = 2093782 }, + { url = "https://files.pythonhosted.org/packages/b3/24/30e33b6389ebb5a17df2a4243b091bc709fb3dfc9a48c8d72f8e037c943d/SQLAlchemy-2.0.36-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f5e9cd989b45b73bd359f693b935364f7e1f79486e29015813c338450aa5a71", size = 2084180 }, + { url = "https://files.pythonhosted.org/packages/10/1e/70e9ed2143a27065246be40f78637ad5160ea0f5fd32f8cab819a31ff54d/SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ddd9db6e59c44875211bc4c7953a9f6638b937b0a88ae6d09eb46cced54eff", size = 3202469 }, + { url = "https://files.pythonhosted.org/packages/b4/5f/95e0ed74093ac3c0db6acfa944d4d8ac6284ef5e1136b878a327ea1f975a/SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2519f3a5d0517fc159afab1015e54bb81b4406c278749779be57a569d8d1bb0d", size = 3202464 }, + { url = "https://files.pythonhosted.org/packages/91/95/2cf9b85a6bc2ee660e40594dffe04e777e7b8617fd0c6d77a0f782ea96c9/SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59b1ee96617135f6e1d6f275bbe988f419c5178016f3d41d3c0abb0c819f75bb", size = 3139508 }, + { url = "https://files.pythonhosted.org/packages/92/ea/f0c01bc646456e4345c0fb5a3ddef457326285c2dc60435b0eb96b61bf31/SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39769a115f730d683b0eb7b694db9789267bcd027326cccc3125e862eb03bfd8", size = 3159837 }, + { url = "https://files.pythonhosted.org/packages/a6/93/c8edbf153ee38fe529773240877bf1332ed95328aceef6254288f446994e/SQLAlchemy-2.0.36-cp311-cp311-win32.whl", hash = "sha256:66bffbad8d6271bb1cc2f9a4ea4f86f80fe5e2e3e501a5ae2a3dc6a76e604e6f", size = 2064529 }, + { url = "https://files.pythonhosted.org/packages/b1/03/d12b7c1d36fd80150c1d52e121614cf9377dac99e5497af8d8f5b2a8db64/SQLAlchemy-2.0.36-cp311-cp311-win_amd64.whl", hash = "sha256:23623166bfefe1487d81b698c423f8678e80df8b54614c2bf4b4cfcd7c711959", size = 2089874 }, + { url = "https://files.pythonhosted.org/packages/b8/bf/005dc47f0e57556e14512d5542f3f183b94fde46e15ff1588ec58ca89555/SQLAlchemy-2.0.36-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7b64e6ec3f02c35647be6b4851008b26cff592a95ecb13b6788a54ef80bbdd4", size = 2092378 }, + { url = "https://files.pythonhosted.org/packages/94/65/f109d5720779a08e6e324ec89a744f5f92c48bd8005edc814bf72fbb24e5/SQLAlchemy-2.0.36-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:46331b00096a6db1fdc052d55b101dbbfc99155a548e20a0e4a8e5e4d1362855", size = 2082778 }, + { url = "https://files.pythonhosted.org/packages/60/f6/d9aa8c49c44f9b8c9b9dada1f12fa78df3d4c42aa2de437164b83ee1123c/SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdf3386a801ea5aba17c6410dd1dc8d39cf454ca2565541b5ac42a84e1e28f53", size = 3232191 }, + { url = "https://files.pythonhosted.org/packages/8a/ab/81d4514527c068670cb1d7ab62a81a185df53a7c379bd2a5636e83d09ede/SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9dfa18ff2a67b09b372d5db8743c27966abf0e5344c555d86cc7199f7ad83a", size = 3243044 }, + { url = "https://files.pythonhosted.org/packages/35/b4/f87c014ecf5167dc669199cafdb20a7358ff4b1d49ce3622cc48571f811c/SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:90812a8933df713fdf748b355527e3af257a11e415b613dd794512461eb8a686", size = 3178511 }, + { url = "https://files.pythonhosted.org/packages/ea/09/badfc9293bc3ccba6ede05e5f2b44a760aa47d84da1fc5a326e963e3d4d9/SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1bc330d9d29c7f06f003ab10e1eaced295e87940405afe1b110f2eb93a233588", size = 3205147 }, + { url = "https://files.pythonhosted.org/packages/c8/60/70e681de02a13c4b27979b7b78da3058c49bacc9858c89ba672e030f03f2/SQLAlchemy-2.0.36-cp312-cp312-win32.whl", hash = "sha256:79d2e78abc26d871875b419e1fd3c0bca31a1cb0043277d0d850014599626c2e", size = 2062709 }, + { url = "https://files.pythonhosted.org/packages/b7/ed/f6cd9395e41bfe47dd253d74d2dfc3cab34980d4e20c8878cb1117306085/SQLAlchemy-2.0.36-cp312-cp312-win_amd64.whl", hash = "sha256:b544ad1935a8541d177cb402948b94e871067656b3a0b9e91dbec136b06a2ff5", size = 2088433 }, + { url = "https://files.pythonhosted.org/packages/78/5c/236398ae3678b3237726819b484f15f5c038a9549da01703a771f05a00d6/SQLAlchemy-2.0.36-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5cc79df7f4bc3d11e4b542596c03826063092611e481fcf1c9dfee3c94355ef", size = 2087651 }, + { url = "https://files.pythonhosted.org/packages/a8/14/55c47420c0d23fb67a35af8be4719199b81c59f3084c28d131a7767b0b0b/SQLAlchemy-2.0.36-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3c01117dd36800f2ecaa238c65365b7b16497adc1522bf84906e5710ee9ba0e8", size = 2078132 }, + { url = "https://files.pythonhosted.org/packages/3d/97/1e843b36abff8c4a7aa2e37f9bea364f90d021754c2de94d792c2d91405b/SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bc633f4ee4b4c46e7adcb3a9b5ec083bf1d9a97c1d3854b92749d935de40b9b", size = 3164559 }, + { url = "https://files.pythonhosted.org/packages/7b/c5/07f18a897b997f6d6b234fab2bf31dccf66d5d16a79fe329aefc95cd7461/SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e46ed38affdfc95d2c958de328d037d87801cfcbea6d421000859e9789e61c2", size = 3177897 }, + { url = "https://files.pythonhosted.org/packages/b3/cd/e16f3cbefd82b5c40b33732da634ec67a5f33b587744c7ab41699789d492/SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b2985c0b06e989c043f1dc09d4fe89e1616aadd35392aea2844f0458a989eacf", size = 3111289 }, + { url = "https://files.pythonhosted.org/packages/15/85/5b8a3b0bc29c9928aa62b5c91fcc8335f57c1de0a6343873b5f372e3672b/SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a121d62ebe7d26fec9155f83f8be5189ef1405f5973ea4874a26fab9f1e262c", size = 3139491 }, + { url = "https://files.pythonhosted.org/packages/a1/95/81babb6089938680dfe2cd3f88cd3fd39cccd1543b7cb603b21ad881bff1/SQLAlchemy-2.0.36-cp313-cp313-win32.whl", hash = "sha256:0572f4bd6f94752167adfd7c1bed84f4b240ee6203a95e05d1e208d488d0d436", size = 2060439 }, + { url = "https://files.pythonhosted.org/packages/c1/ce/5f7428df55660d6879d0522adc73a3364970b5ef33ec17fa125c5dbcac1d/SQLAlchemy-2.0.36-cp313-cp313-win_amd64.whl", hash = "sha256:8c78ac40bde930c60e0f78b3cd184c580f89456dd87fc08f9e3ee3ce8765ce88", size = 2084574 }, + { url = "https://files.pythonhosted.org/packages/b8/49/21633706dd6feb14cd3f7935fc00b60870ea057686035e1a99ae6d9d9d53/SQLAlchemy-2.0.36-py3-none-any.whl", hash = "sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e", size = 1883787 }, ] [[package]] @@ -2845,6 +2872,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826 }, ] +[[package]] +name = "win32-setctime" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/dd/f95a13d2b235a28d613ba23ebad55191514550debb968b46aab99f2e3a30/win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2", size = 3676 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/e6/a7d828fef907843b2a5773ebff47fb79ac0c1c88d60c0ca9530ee941e248/win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad", size = 3604 }, +] + [[package]] name = "wrapt" version = "1.16.0" From dc606b667a66fee8226d499f78c42bbc0b63dd23 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 8 Nov 2024 11:40:38 +1100 Subject: [PATCH 07/12] chore: Update configuration --- packages/ref/src/ref/config.py | 12 ++++++++++-- packages/ref/src/ref/database.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/ref/src/ref/config.py b/packages/ref/src/ref/config.py index 9aa8333..c59321e 100644 --- a/packages/ref/src/ref/config.py +++ b/packages/ref/src/ref/config.py @@ -5,7 +5,7 @@ # https://github.com/ESGF/esgf-download/blob/main/esgpull/config.py from pathlib import Path -from typing import Any +from typing import Any, Literal import tomlkit from attrs import Factory, define, field @@ -61,7 +61,15 @@ class Db: Database configuration """ - filename: str = "ref.db" + engine: Literal["sqlite", "postgres"] = field(default="sqlite") + connection_url: str = field() + run_migrations: bool = field(default=True) + + @connection_url.default + def _connection_url_factory(self) -> str: + filename = env.path("REF_CONFIGURATION") / "db" / "ref.db" + sqlite_url = f"sqlite:///{filename}" + return sqlite_url @define diff --git a/packages/ref/src/ref/database.py b/packages/ref/src/ref/database.py index b8ad352..6f9b82a 100644 --- a/packages/ref/src/ref/database.py +++ b/packages/ref/src/ref/database.py @@ -57,7 +57,7 @@ def from_config(config: Config, run_migrations: bool = True) -> "Database": """ # TODO: move the database URL creation to the Config class config.paths.db.mkdir(parents=True, exist_ok=True) - url = f"sqlite:///{config.paths.db / config.db.filename}" + url = config.db.connection_url return Database(url, run_migrations=run_migrations) From 0b7fa637bfecd5c39a8fb5422fd7b306c26397b6 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 8 Nov 2024 12:26:32 +1100 Subject: [PATCH 08/12] feat: validate database urls --- packages/ref/pyproject.toml | 5 ++ packages/ref/src/ref/config.py | 26 +++++---- packages/ref/src/ref/database.py | 57 +++++++++++++++++--- packages/ref/tests/conftest.py | 4 ++ packages/ref/tests/unit/test_database.py | 67 ++++++++++++++++++++++++ pyproject.toml | 2 +- uv.lock | 64 +++++++++++++++++++++- 7 files changed, 205 insertions(+), 20 deletions(-) create mode 100644 packages/ref/tests/unit/test_database.py diff --git a/packages/ref/pyproject.toml b/packages/ref/pyproject.toml index 55962ea..fe413fb 100644 --- a/packages/ref/pyproject.toml +++ b/packages/ref/pyproject.toml @@ -32,6 +32,11 @@ dependencies = [ "loguru>=0.7.2", ] +[project.optional-dependencies] +postgres = [ + "psycopg2-binary>=2.9.2", +] + [project.scripts] ref = "ref.cli:app" diff --git a/packages/ref/src/ref/config.py b/packages/ref/src/ref/config.py index c59321e..ddbc211 100644 --- a/packages/ref/src/ref/config.py +++ b/packages/ref/src/ref/config.py @@ -5,7 +5,7 @@ # https://github.com/ESGF/esgf-download/blob/main/esgpull/config.py from pathlib import Path -from typing import Any, Literal +from typing import Any import tomlkit from attrs import Factory, define, field @@ -34,7 +34,6 @@ class Paths: """ data: Path = field(converter=Path) - db: Path = field(converter=Path) log: Path = field(converter=Path) tmp: Path = field(converter=Path) @@ -42,10 +41,6 @@ class Paths: def _data_factory(self) -> Path: return env.path("REF_CONFIGURATION") / "data" - @db.default - def _db_factory(self) -> Path: - return env.path("REF_CONFIGURATION") / "db" - @log.default def _log_factory(self) -> Path: return env.path("REF_CONFIGURATION") / "log" @@ -59,13 +54,26 @@ def _tmp_factory(self) -> Path: class Db: """ Database configuration + + We currently only plan to support SQLite and PostgreSQL databases, + although only SQLite is currently implemented and tested. + """ + + database_url: str = field() """ + Database URL that describes the connection to the database. - engine: Literal["sqlite", "postgres"] = field(default="sqlite") - connection_url: str = field() + Defaults to sqlite:///{config.paths.db}/ref.db". + This configuration value will be overridden by the `REF_DATABASE_URL` environment variable. + + ## Schemas + + postgresql://USER:PASSWORD@HOST:PORT/NAME + sqlite:///RELATIVE_PATH or sqlite:////ABS_PATH or sqlite:///:memory: + """ run_migrations: bool = field(default=True) - @connection_url.default + @database_url.default def _connection_url_factory(self) -> str: filename = env.path("REF_CONFIGURATION") / "db" / "ref.db" sqlite_url = f"sqlite:///{filename}" diff --git a/packages/ref/src/ref/database.py b/packages/ref/src/ref/database.py index 6f9b82a..94932a9 100644 --- a/packages/ref/src/ref/database.py +++ b/packages/ref/src/ref/database.py @@ -1,4 +1,5 @@ from pathlib import Path +from urllib import parse as urlparse import alembic.command import sqlalchemy @@ -8,6 +9,48 @@ from sqlalchemy.orm import Session from ref.config import Config +from ref.env import env + + +def validate_database_url(database_url: str): + """ + Validate a database URL + + We support sqlite databases, and we create the directory if it doesn't exist. + + Parameters + ---------- + database_url + The database URL to validate + + See [ref.config.Db.database_url](ref.config.Db.database_url) for more information + on the format of the URL. + + Raises + ------ + ValueError + If the database scheme is not supported + + Returns + ------- + : + The validated database URL + """ + split_url = urlparse.urlsplit(database_url) + path = split_url.path[1:] + + if split_url.scheme == "sqlite": + if path == ":memory:": + logger.warning("Using an in-memory database") + else: + Path(path).parent.mkdir(parents=True, exist_ok=True) + elif split_url.scheme == "postgresql": + # We don't need to do anything special for PostgreSQL + logger.warning("PostgreSQL support is currently experimental and untested") + else: + raise ValueError(f"Unsupported database scheme: {split_url.scheme}") + + return database_url class Database: @@ -43,6 +86,9 @@ def from_config(config: Config, run_migrations: bool = True) -> "Database": """ Create a Database instance from a Config instance + The `REF_DATABASE_URL` environment variable will take preference, + and override the database URL specified in the config. + Parameters ---------- config @@ -55,12 +101,7 @@ def from_config(config: Config, run_migrations: bool = True) -> "Database": : A new Database instance """ - # TODO: move the database URL creation to the Config class - config.paths.db.mkdir(parents=True, exist_ok=True) - url = config.db.connection_url - - return Database(url, run_migrations=run_migrations) - + database_url: str = env.str("REF_DATABASE_URL", default=config.db.database_url) -if __name__ == "__main__": - Database.from_config(Config(), run_migrations=True) + database_url = validate_database_url(database_url) + return Database(database_url, run_migrations=run_migrations) diff --git a/packages/ref/tests/conftest.py b/packages/ref/tests/conftest.py index 9673afb..696be94 100644 --- a/packages/ref/tests/conftest.py +++ b/packages/ref/tests/conftest.py @@ -1,4 +1,5 @@ import pytest + from ref.config import Config @@ -8,6 +9,9 @@ def config(tmp_path, monkeypatch) -> Config: # Uses the default configuration cfg = Config.load(tmp_path / "ref" / "ref.toml") + + # Use a SQLite in-memory database for testing + cfg.db.database_url = "sqlite:///:memory:" cfg.save() return cfg diff --git a/packages/ref/tests/unit/test_database.py b/packages/ref/tests/unit/test_database.py new file mode 100644 index 0000000..7b00f84 --- /dev/null +++ b/packages/ref/tests/unit/test_database.py @@ -0,0 +1,67 @@ +import pytest +import sqlalchemy +from sqlalchemy import select + +from ref.database import Database, validate_database_url +from ref.models import Dataset + + +@pytest.mark.parametrize( + "database_url", + [ + "sqlite:///:memory:", + "sqlite:///{tmp_path}/ref.db", + "postgresql://localhost:5432/ref", + ], +) +def test_validate_database_url(config, database_url, tmp_path): + validate_database_url(database_url.format(tmp_path=str(tmp_path))) + + +@pytest.mark.parametrize("database_url", ["mysql:///:memory:", "no_scheme/test"]) +def test_invalid_urls(config, database_url, tmp_path): + with pytest.raises(ValueError): + validate_database_url(database_url.format(tmp_path=str(tmp_path))) + + +def test_database(config): + db = Database.from_config(config, run_migrations=True) + + assert db._engine + assert db.session.is_active + + db.session.add( + Dataset( + dataset_id="big_dataset", + instance_id="test", + master_id="master", + version="version", + data_node="data_node", + size=12, + number_of_files=1, + ) + ) + db.session.add( + Dataset( + dataset_id="small_dataset", + instance_id="test", + master_id="master", + version="version", + data_node="data_node", + size=1, + number_of_files=1, + ) + ) + + stmt = select(Dataset).where(Dataset.size >= 12) + res = db.session.scalars(stmt).all() + assert len(res) == 1 + + assert res[0].dataset_id == "big_dataset" + + +def test_database_invalid_url(config, monkeypatch): + monkeypatch.setenv("REF_DATABASE_URL", "postgresql:///localhost:12323/ref") + + with pytest.raises(sqlalchemy.exc.OperationalError): + Database.from_config(config, run_migrations=True) diff --git a/pyproject.toml b/pyproject.toml index dabebb7..295f608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [ ] requires-python = ">=3.10" dependencies = [ - "ref", + "ref[postgres]", "ref-core", "ref-metrics-example", ] diff --git a/uv.lock b/uv.lock index 2a14c5e..c4a1ee2 100644 --- a/uv.lock +++ b/uv.lock @@ -364,7 +364,7 @@ name = "cmip-ref" version = "0.1.0" source = { virtual = "." } dependencies = [ - { name = "ref" }, + { name = "ref", extra = ["postgres"] }, { name = "ref-core" }, { name = "ref-metrics-example" }, ] @@ -395,7 +395,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "ref", editable = "packages/ref" }, + { name = "ref", extras = ["postgres"], editable = "packages/ref" }, { name = "ref-core", editable = "packages/ref-core" }, { name = "ref-metrics-example", editable = "packages/ref-metrics-example" }, ] @@ -1851,6 +1851,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/06/63872a64c312a24fb9b4af123ee7007a306617da63ff13bcc1432386ead7/psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0", size = 251988 }, ] +[[package]] +name = "psycopg2-binary" +version = "2.9.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/bdc8274dc0585090b4e3432267d7be4dfbfd8971c0fa59167c711105a6bf/psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2", size = 385764 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/81/331257dbf2801cdb82105306042f7a1637cc752f65f2bb688188e0de5f0b/psycopg2_binary-2.9.10-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f", size = 3043397 }, + { url = "https://files.pythonhosted.org/packages/e7/9a/7f4f2f031010bbfe6a02b4a15c01e12eb6b9b7b358ab33229f28baadbfc1/psycopg2_binary-2.9.10-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906", size = 3274806 }, + { url = "https://files.pythonhosted.org/packages/e5/57/8ddd4b374fa811a0b0a0f49b6abad1cde9cb34df73ea3348cc283fcd70b4/psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92", size = 2851361 }, + { url = "https://files.pythonhosted.org/packages/f9/66/d1e52c20d283f1f3a8e7e5c1e06851d432f123ef57b13043b4f9b21ffa1f/psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007", size = 3080836 }, + { url = "https://files.pythonhosted.org/packages/a0/cb/592d44a9546aba78f8a1249021fe7c59d3afb8a0ba51434d6610cc3462b6/psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0", size = 3264552 }, + { url = "https://files.pythonhosted.org/packages/64/33/c8548560b94b7617f203d7236d6cdf36fe1a5a3645600ada6efd79da946f/psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4", size = 3019789 }, + { url = "https://files.pythonhosted.org/packages/b0/0e/c2da0db5bea88a3be52307f88b75eec72c4de62814cbe9ee600c29c06334/psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1", size = 2871776 }, + { url = "https://files.pythonhosted.org/packages/15/d7/774afa1eadb787ddf41aab52d4c62785563e29949613c958955031408ae6/psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5", size = 2820959 }, + { url = "https://files.pythonhosted.org/packages/5e/ed/440dc3f5991a8c6172a1cde44850ead0e483a375277a1aef7cfcec00af07/psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5", size = 2919329 }, + { url = "https://files.pythonhosted.org/packages/03/be/2cc8f4282898306732d2ae7b7378ae14e8df3c1231b53579efa056aae887/psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53", size = 2957659 }, + { url = "https://files.pythonhosted.org/packages/d0/12/fb8e4f485d98c570e00dad5800e9a2349cfe0f71a767c856857160d343a5/psycopg2_binary-2.9.10-cp310-cp310-win32.whl", hash = "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b", size = 1024605 }, + { url = "https://files.pythonhosted.org/packages/22/4f/217cd2471ecf45d82905dd09085e049af8de6cfdc008b6663c3226dc1c98/psycopg2_binary-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1", size = 1163817 }, + { url = "https://files.pythonhosted.org/packages/9c/8f/9feb01291d0d7a0a4c6a6bab24094135c2b59c6a81943752f632c75896d6/psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff", size = 3043397 }, + { url = "https://files.pythonhosted.org/packages/15/30/346e4683532011561cd9c8dfeac6a8153dd96452fee0b12666058ab7893c/psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c", size = 3274806 }, + { url = "https://files.pythonhosted.org/packages/66/6e/4efebe76f76aee7ec99166b6c023ff8abdc4e183f7b70913d7c047701b79/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c", size = 2851370 }, + { url = "https://files.pythonhosted.org/packages/7f/fd/ff83313f86b50f7ca089b161b8e0a22bb3c319974096093cd50680433fdb/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb", size = 3080780 }, + { url = "https://files.pythonhosted.org/packages/e6/c4/bfadd202dcda8333a7ccafdc51c541dbdfce7c2c7cda89fa2374455d795f/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341", size = 3264583 }, + { url = "https://files.pythonhosted.org/packages/5d/f1/09f45ac25e704ac954862581f9f9ae21303cc5ded3d0b775532b407f0e90/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a", size = 3019831 }, + { url = "https://files.pythonhosted.org/packages/9e/2e/9beaea078095cc558f215e38f647c7114987d9febfc25cb2beed7c3582a5/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b", size = 2871822 }, + { url = "https://files.pythonhosted.org/packages/01/9e/ef93c5d93f3dc9fc92786ffab39e323b9aed066ba59fdc34cf85e2722271/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7", size = 2820975 }, + { url = "https://files.pythonhosted.org/packages/a5/f0/049e9631e3268fe4c5a387f6fc27e267ebe199acf1bc1bc9cbde4bd6916c/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e", size = 2919320 }, + { url = "https://files.pythonhosted.org/packages/dc/9a/bcb8773b88e45fb5a5ea8339e2104d82c863a3b8558fbb2aadfe66df86b3/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68", size = 2957617 }, + { url = "https://files.pythonhosted.org/packages/e2/6b/144336a9bf08a67d217b3af3246abb1d027095dab726f0687f01f43e8c03/psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392", size = 1024618 }, + { url = "https://files.pythonhosted.org/packages/61/69/3b3d7bd583c6d3cbe5100802efa5beacaacc86e37b653fc708bf3d6853b8/psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4", size = 1163816 }, + { url = "https://files.pythonhosted.org/packages/49/7d/465cc9795cf76f6d329efdafca74693714556ea3891813701ac1fee87545/psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0", size = 3044771 }, + { url = "https://files.pythonhosted.org/packages/8b/31/6d225b7b641a1a2148e3ed65e1aa74fc86ba3fee850545e27be9e1de893d/psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a", size = 3275336 }, + { url = "https://files.pythonhosted.org/packages/30/b7/a68c2b4bff1cbb1728e3ec864b2d92327c77ad52edcd27922535a8366f68/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539", size = 2851637 }, + { url = "https://files.pythonhosted.org/packages/0b/b1/cfedc0e0e6f9ad61f8657fd173b2f831ce261c02a08c0b09c652b127d813/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526", size = 3082097 }, + { url = "https://files.pythonhosted.org/packages/18/ed/0a8e4153c9b769f59c02fb5e7914f20f0b2483a19dae7bf2db54b743d0d0/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1", size = 3264776 }, + { url = "https://files.pythonhosted.org/packages/10/db/d09da68c6a0cdab41566b74e0a6068a425f077169bed0946559b7348ebe9/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e", size = 3020968 }, + { url = "https://files.pythonhosted.org/packages/94/28/4d6f8c255f0dfffb410db2b3f9ac5218d959a66c715c34cac31081e19b95/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f", size = 2872334 }, + { url = "https://files.pythonhosted.org/packages/05/f7/20d7bf796593c4fea95e12119d6cc384ff1f6141a24fbb7df5a668d29d29/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00", size = 2822722 }, + { url = "https://files.pythonhosted.org/packages/4d/e4/0c407ae919ef626dbdb32835a03b6737013c3cc7240169843965cada2bdf/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5", size = 2920132 }, + { url = "https://files.pythonhosted.org/packages/2d/70/aa69c9f69cf09a01da224909ff6ce8b68faeef476f00f7ec377e8f03be70/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47", size = 2959312 }, + { url = "https://files.pythonhosted.org/packages/d3/bd/213e59854fafe87ba47814bf413ace0dcee33a89c8c8c814faca6bc7cf3c/psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64", size = 1025191 }, + { url = "https://files.pythonhosted.org/packages/92/29/06261ea000e2dc1e22907dbbc483a1093665509ea586b29b8986a0e56733/psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0", size = 1164031 }, + { url = "https://files.pythonhosted.org/packages/3e/30/d41d3ba765609c0763505d565c4d12d8f3c79793f0d0f044ff5a28bf395b/psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d", size = 3044699 }, + { url = "https://files.pythonhosted.org/packages/35/44/257ddadec7ef04536ba71af6bc6a75ec05c5343004a7ec93006bee66c0bc/psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb", size = 3275245 }, + { url = "https://files.pythonhosted.org/packages/1b/11/48ea1cd11de67f9efd7262085588790a95d9dfcd9b8a687d46caf7305c1a/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7", size = 2851631 }, + { url = "https://files.pythonhosted.org/packages/62/e0/62ce5ee650e6c86719d621a761fe4bc846ab9eff8c1f12b1ed5741bf1c9b/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d", size = 3082140 }, + { url = "https://files.pythonhosted.org/packages/27/ce/63f946c098611f7be234c0dd7cb1ad68b0b5744d34f68062bb3c5aa510c8/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73", size = 3264762 }, + { url = "https://files.pythonhosted.org/packages/43/25/c603cd81402e69edf7daa59b1602bd41eb9859e2824b8c0855d748366ac9/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673", size = 3020967 }, + { url = "https://files.pythonhosted.org/packages/5f/d6/8708d8c6fca531057fa170cdde8df870e8b6a9b136e82b361c65e42b841e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f", size = 2872326 }, + { url = "https://files.pythonhosted.org/packages/ce/ac/5b1ea50fc08a9df82de7e1771537557f07c2632231bbab652c7e22597908/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909", size = 2822712 }, + { url = "https://files.pythonhosted.org/packages/c4/fc/504d4503b2abc4570fac3ca56eb8fed5e437bf9c9ef13f36b6621db8ef00/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1", size = 2920155 }, + { url = "https://files.pythonhosted.org/packages/b2/d1/323581e9273ad2c0dbd1902f3fb50c441da86e894b6e25a73c3fda32c57e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567", size = 2959356 }, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -2156,6 +2210,11 @@ dependencies = [ { name = "typer" }, ] +[package.optional-dependencies] +postgres = [ + { name = "psycopg2-binary" }, +] + [package.metadata] requires-dist = [ { name = "alembic", specifier = ">=1.13.3" }, @@ -2163,6 +2222,7 @@ requires-dist = [ { name = "cattrs", specifier = ">=24.1.2" }, { name = "environs", specifier = ">=11.0.0" }, { name = "loguru", specifier = ">=0.7.2" }, + { name = "psycopg2-binary", marker = "extra == 'postgres'", specifier = ">=2.9.2" }, { name = "ref-core", editable = "packages/ref-core" }, { name = "sqlalchemy", specifier = ">=2.0.36" }, { name = "tomlkit", specifier = ">=0.13.2" }, From 1e89c2d22e8b68b0ac79b9a74e73ba7ebac97cb6 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 8 Nov 2024 12:29:24 +1100 Subject: [PATCH 09/12] chore: Fix mypy --- packages/ref/src/ref/database.py | 6 +++--- packages/ref/src/ref/models/dataset.py | 2 +- stubs/environs/__init__.pyi | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/ref/src/ref/database.py b/packages/ref/src/ref/database.py index 94932a9..b058338 100644 --- a/packages/ref/src/ref/database.py +++ b/packages/ref/src/ref/database.py @@ -12,7 +12,7 @@ from ref.env import env -def validate_database_url(database_url: str): +def validate_database_url(database_url: str) -> str: """ Validate a database URL @@ -68,7 +68,7 @@ def __init__(self, url: str, run_migrations: bool = True) -> None: if run_migrations: self._migrate() - def _migrate(self): + def _migrate(self) -> None: root_dir = Path(__file__).parents[4] alembic_config = AlembicConfig(root_dir / "alembic.ini") @@ -79,7 +79,7 @@ def _migrate(self): head = script.get_current_head() # Run migrations - alembic.command.upgrade(alembic_config, head) + alembic.command.upgrade(alembic_config, head or "heads") @staticmethod def from_config(config: Config, run_migrations: bool = True) -> "Database": diff --git a/packages/ref/src/ref/models/dataset.py b/packages/ref/src/ref/models/dataset.py index f4bf76b..d9bab9b 100644 --- a/packages/ref/src/ref/models/dataset.py +++ b/packages/ref/src/ref/models/dataset.py @@ -42,5 +42,5 @@ class Dataset(Base): # variable_id, table_id, institution_id, model_id, experiment_id, source_id, member_id, grid_label, # time_range, time_frequency, realm,retracted - def __repr__(self): + def __repr__(self) -> str: return f"" diff --git a/stubs/environs/__init__.pyi b/stubs/environs/__init__.pyi index 9f4ae8c..bdb6922 100644 --- a/stubs/environs/__init__.pyi +++ b/stubs/environs/__init__.pyi @@ -2,7 +2,6 @@ from pathlib import Path class Env: def __init__(self, expand_vars: bool) -> None: ... - def path(self, name: str) -> Path: ... def read_env( self, path: str | None = None, @@ -10,3 +9,5 @@ class Env: verbose: bool = False, override: bool = False, ) -> bool: ... + def path(self, name: str, default: str | None = None) -> Path: ... + def str(self, name: str, default: str | None = None) -> str: ... From 4752640d7e951f7dcb16da669e21f18847e04b44 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 8 Nov 2024 12:35:03 +1100 Subject: [PATCH 10/12] test: Fix tests --- packages/ref/{tests => }/conftest.py | 3 +++ packages/ref/tests/unit/cli/test_config.py | 5 +++-- packages/ref/tests/unit/test_config.py | 6 +++--- pyproject.toml | 2 ++ 4 files changed, 11 insertions(+), 5 deletions(-) rename packages/ref/{tests => }/conftest.py (87%) diff --git a/packages/ref/tests/conftest.py b/packages/ref/conftest.py similarity index 87% rename from packages/ref/tests/conftest.py rename to packages/ref/conftest.py index 696be94..f7f4a28 100644 --- a/packages/ref/tests/conftest.py +++ b/packages/ref/conftest.py @@ -2,6 +2,9 @@ from ref.config import Config +# Ignore the alembic folder +collect_ignore = ["alembic"] + @pytest.fixture def config(tmp_path, monkeypatch) -> Config: diff --git a/packages/ref/tests/unit/cli/test_config.py b/packages/ref/tests/unit/cli/test_config.py index 8423a9f..cdb6593 100644 --- a/packages/ref/tests/unit/cli/test_config.py +++ b/packages/ref/tests/unit/cli/test_config.py @@ -1,7 +1,8 @@ import platformdirs -from ref.cli import app from typer.testing import CliRunner +from ref.cli import app + runner = CliRunner() @@ -25,7 +26,7 @@ def test_config_list(self): config_dir = platformdirs.user_config_dir("cmip-ref") assert f'data = "{config_dir}/data"\n' in result.output - assert 'filename = "sqlite://ref.db"\n' in result.output + assert 'database_url = "sqlite://' in result.output def test_config_list_custom(self, config): result = runner.invoke( diff --git a/packages/ref/tests/unit/test_config.py b/packages/ref/tests/unit/test_config.py index 81ec7b4..a21ede6 100644 --- a/packages/ref/tests/unit/test_config.py +++ b/packages/ref/tests/unit/test_config.py @@ -2,6 +2,7 @@ import cattrs import pytest + from ref.config import Config, Paths @@ -16,7 +17,6 @@ def test_load_missing(self, tmp_path, monkeypatch): loaded = Config.load(Path("ref.toml")) assert loaded.paths.data == tmp_path / "ref" / "data" - assert loaded.paths.db == tmp_path / "ref" / "db" # The results aren't serialised back to disk assert not (tmp_path / "ref.toml").exists() @@ -92,6 +92,6 @@ def test_defaults(self, monkeypatch): assert without_defaults == {} assert with_defaults == { - "paths": {"data": "test/data", "db": "test/db", "log": "test/log", "tmp": "test/tmp"}, - "db": {"filename": "sqlite://ref.db"}, + "paths": {"data": "test/data", "log": "test/log", "tmp": "test/tmp"}, + "db": {"database_url": "sqlite:///test/db/ref.db", "run_migrations": True}, } diff --git a/pyproject.toml b/pyproject.toml index 295f608..9a5de48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ warn_unreachable = true # importing following uses default settings follow_imports = "normal" exclude = [ + "alembic", "build", "dist", "notebooks", @@ -101,6 +102,7 @@ filterwarnings = [ ] + # We currently check for GPL licensed code, but this restriction may be removed [tool.liccheck] authorized_licenses = [ From a88968e28d8b150b1518439eed4ce426b6f6c1f0 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 8 Nov 2024 13:44:29 +1100 Subject: [PATCH 11/12] chore: Fix mypy --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9a5de48..c00dea1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ exclude = [ "scripts", "stubs", "tests", + "conftest.py" ] From 86c856cb1e4b6375abb475b0ed649fd15635261f Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 8 Nov 2024 13:45:40 +1100 Subject: [PATCH 12/12] docs: Changelog --- changelog/11.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/11.feature.md diff --git a/changelog/11.feature.md b/changelog/11.feature.md new file mode 100644 index 0000000..1d7b00f --- /dev/null +++ b/changelog/11.feature.md @@ -0,0 +1 @@ +Add `SqlAlchemy` as an ORM for the database alongside `alembic` for managing database migrations.