diff --git a/packages/ref/pyproject.toml b/packages/ref/pyproject.toml index 7b0ad4e..82b51aa 100644 --- a/packages/ref/pyproject.toml +++ b/packages/ref/pyproject.toml @@ -21,6 +21,10 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dependencies = [ + "attrs>=24.2.0", + "cattrs>=24.1.2", + "environs>=11.0.0", + "tomlkit>=0.13.2", "typer>=0.12.5", ] diff --git a/packages/ref/src/ref/cli/__init__.py b/packages/ref/src/ref/cli/__init__.py index dfc84b9..95e7c4d 100644 --- a/packages/ref/src/ref/cli/__init__.py +++ b/packages/ref/src/ref/cli/__init__.py @@ -4,7 +4,7 @@ from ref.cli import config, sync -app = typer.Typer() +app = typer.Typer(name="ref") app.command(name="sync")(sync.sync) app.add_typer(config.app, name="config") diff --git a/packages/ref/src/ref/cli/config.py b/packages/ref/src/ref/cli/config.py index fa95e71..f729a74 100644 --- a/packages/ref/src/ref/cli/config.py +++ b/packages/ref/src/ref/cli/config.py @@ -2,21 +2,39 @@ View and update the REF configuration """ +from pathlib import Path + import typer +from ref.config import Config +from ref.constants import config_filename + app = typer.Typer(help=__doc__) @app.command() -def list(): +def list(configuration_directory: Path | None = typer.Option(None, help="Configuration directory")) -> None: """ Print the current ref configuration + + If a configuration directory is provided, + the configuration will attempted to be loaded from the specified directory. + If the configuration file is missing then a """ - print("config") + try: + if configuration_directory: + config = Config.load(configuration_directory / config_filename, allow_missing=False) + else: + config = Config.default() + except FileNotFoundError: + typer.secho("Configuration file not found", fg=typer.colors.RED) + raise typer.Exit(1) + + print(config.dumps(defaults=True)) @app.command() -def update(): +def update() -> None: """ Print the current ref configuration """ diff --git a/packages/ref/src/ref/cli/sync.py b/packages/ref/src/ref/cli/sync.py index ad4f34f..ce7b2f1 100644 --- a/packages/ref/src/ref/cli/sync.py +++ b/packages/ref/src/ref/cli/sync.py @@ -4,7 +4,7 @@ @app.command() -def sync(): +def sync() -> None: """ Placeholder command for syncing data """ # noqa: D401 diff --git a/packages/ref/src/ref/config.py b/packages/ref/src/ref/config.py new file mode 100644 index 0000000..6b3c85c --- /dev/null +++ b/packages/ref/src/ref/config.py @@ -0,0 +1,218 @@ +"""Configuration management""" + +# The basics of the configuration management takes a lot of inspiration from the +# `esgpull` configuration management system with some of the extra complexity removed. +# https://github.com/ESGF/esgf-download/blob/main/esgpull/config.py + +from pathlib import Path +from typing import Any + +import tomlkit +from attrs import Factory, define, field +from cattrs import Converter +from cattrs.gen import make_dict_unstructure_fn, override +from tomlkit import TOMLDocument + +from ref.constants import config_filename +from ref.env import env + + +def _pop_empty(d: dict[str, Any]) -> None: + keys = list(d.keys()) + for key in keys: + value = d[key] + if isinstance(value, dict): + _pop_empty(value) + if not value: + d.pop(key) + + +@define +class Paths: + """ + Common paths used by the REF application + """ + + data: Path = field(converter=Path) + db: Path = field(converter=Path) + log: Path = field(converter=Path) + tmp: Path = field(converter=Path) + + @data.default + def _data_factory(self) -> Path: + return env.path("REF_CONFIGURATION") / "data" + + @db.default + def _db_factory(self) -> Path: + return env.path("REF_CONFIGURATION") / "db" + + @log.default + def _log_factory(self) -> Path: + return env.path("REF_CONFIGURATION") / "log" + + @tmp.default + def _tmp_factory(self) -> Path: + return env.path("REF_CONFIGURATION") / "tmp" + + +@define +class Db: + """ + Database configuration + """ + + filename: str = "sqlite://ref.db" + + +@define +class Config: + """ + REF configuration + + This class is used to store the configuration of the REF application. + """ + + paths: Paths = Factory(Paths) + db: Db = Factory(Db) + _raw: TOMLDocument | None = field(init=False, default=None) + _config_file: Path | None = field(init=False, default=None) + + @classmethod + def load(cls, config_file: Path, allow_missing: bool = True) -> "Config": + """ + Load the configuration from a file + + Parameters + ---------- + config_file + Path to the configuration file. + This should be a TOML file. + + Returns + ------- + : + The configuration loaded from the file + """ + if config_file.is_file(): + with config_file.open() as fh: + doc = tomlkit.load(fh) + raw = doc + else: + if not allow_missing: + raise FileNotFoundError(f"Configuration file not found: {config_file}") + + doc = TOMLDocument() + raw = None + config = _converter_defaults.structure(doc, cls) + config._raw = raw + config._config_file = config_file + return config + + def save(self, config_file: Path | None = None) -> None: + """ + Save the configuration as a TOML file + + The configuration will be saved to the specified file. + If no file is specified, the configuration will be saved to the file + that was used to load the configuration. + + Parameters + ---------- + config_file + The file to save the configuration to + + Raises + ------ + ValueError + If no configuration file is specified and the configuration was not loaded from a file + """ + if config_file is None: + if self._config_file is None: # pragma: no cover + # I'm not sure if this is possible + raise ValueError("No configuration file specified") + config_file = self._config_file + + config_file.parent.mkdir(parents=True, exist_ok=True) + + with open(config_file, "w") as fh: + fh.write(self.dumps()) + + @classmethod + def default(cls) -> "Config": + """ + Load the default configuration + + This will load the configuration from the default configuration location, + which is typically the user's configuration directory. + This location can be overridden by setting the `REF_CONFIGURATION` environment variable. + + Returns + ------- + : + The default configuration + """ + root = env.path("REF_CONFIGURATION") + return cls.load(root / config_filename) + + def dumps(self, defaults: bool = True) -> str: + """ + Dump the configuration to a TOML string + + Parameters + ---------- + defaults + If True, include default values in the output + + Returns + ------- + : + The configuration as a TOML string + """ + return self.dump(defaults).as_string() + + def dump( + self, + defaults: bool = True, + ) -> TOMLDocument: + """ + Dump the configuration to a TOML document + + Parameters + ---------- + defaults + If True, include default values in the output + + Returns + ------- + : + The configuration as a TOML document + """ + if defaults: + converter = _converter_defaults + else: + converter = _converter_no_defaults + dump = converter.unstructure(self) + if not defaults: + _pop_empty(dump) + doc = TOMLDocument() + doc.update(dump) + return doc + + +def _make_converter(omit_default: bool) -> Converter: + conv = Converter(omit_if_default=omit_default, forbid_extra_keys=True) + conv.register_unstructure_hook(Path, str) + conv.register_unstructure_hook( + Config, + make_dict_unstructure_fn( + Config, + conv, + _raw=override(omit=True), + _config_file=override(omit=True), + ), + ) + return conv + + +_converter_defaults = _make_converter(omit_default=False) +_converter_no_defaults = _make_converter(omit_default=True) diff --git a/packages/ref/src/ref/constants.py b/packages/ref/src/ref/constants.py new file mode 100644 index 0000000..1eb2211 --- /dev/null +++ b/packages/ref/src/ref/constants.py @@ -0,0 +1,8 @@ +""" +Constants used by the REF +""" + +config_filename = "ref.toml" +""" +Default name of the configuration file +""" diff --git a/packages/ref/src/ref/env.py b/packages/ref/src/ref/env.py new file mode 100644 index 0000000..4cb9782 --- /dev/null +++ b/packages/ref/src/ref/env.py @@ -0,0 +1,35 @@ +"""Environment variable management""" + +import os + +import platformdirs +from environs import Env + + +def _set_defaults() -> None: + os.environ.setdefault("REF_CONFIGURATION", str(platformdirs.user_config_path("cmip-ref"))) + + +def get_env() -> Env: + """ + Get the current environment + + Returns + ------- + : + The current environment including any environment variables loaded from the .env file + and any defaults set by this application. + """ + # Set the default values for the environment variables + _set_defaults() + + env = Env(expand_vars=True) + + # Load the environment variables from the .env file + # This will override any defaults set above + env.read_env(verbose=True) + + return env + + +env = get_env() diff --git a/packages/ref/tests/conftest.py b/packages/ref/tests/conftest.py new file mode 100644 index 0000000..9673afb --- /dev/null +++ b/packages/ref/tests/conftest.py @@ -0,0 +1,13 @@ +import pytest +from ref.config import Config + + +@pytest.fixture +def config(tmp_path, monkeypatch) -> Config: + monkeypatch.setenv("REF_CONFIGURATION", str(tmp_path / "ref")) + + # Uses the default configuration + cfg = Config.load(tmp_path / "ref" / "ref.toml") + cfg.save() + + return cfg diff --git a/packages/ref/tests/unit/cli/test_config.py b/packages/ref/tests/unit/cli/test_config.py new file mode 100644 index 0000000..29311f0 --- /dev/null +++ b/packages/ref/tests/unit/cli/test_config.py @@ -0,0 +1,60 @@ +import platformdirs +from ref.cli import app +from typer.testing import CliRunner + +runner = CliRunner() + + +def test_without_subcommand(): + result = runner.invoke(app, ["config"]) + assert result.exit_code == 2 + assert "Missing command." in result.output + + +def test_config_help(): + result = runner.invoke(app, ["config", "--help"]) + assert result.exit_code == 0 + + expected = """ + Usage: ref config [OPTIONS] COMMAND [ARGS]... + + View and update the REF configuration + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────╮ +│ list Print the current ref configuration │ +│ update Print the current ref configuration │ +╰──────────────────────────────────────────────────────────────────────────────╯ +""" + assert expected in result.output + + +class TestConfigList: + def test_config_list(self): + result = runner.invoke(app, ["config", "list"]) + assert result.exit_code == 0 + + config_dir = platformdirs.user_config_dir("cmip-ref") + assert f'data = "{config_dir}/data"\n' in result.output + assert 'filename = "sqlite://ref.db"\n' in result.output + + def test_config_list_custom(self, config): + result = runner.invoke( + app, ["config", "list", "--configuration-directory", str(config._config_file.parent)] + ) + assert result.exit_code == 0 + + def test_config_list_custom_missing(self, config): + result = runner.invoke(app, ["config", "list", "--configuration-directory", "missing"]) + assert result.exit_code == 1 + + +class TestConfigUpdate: + def test_config_update(self): + result = runner.invoke(app, ["config", "update"]) + assert result.exit_code == 0 + + # TODO: actually implement this functionality + assert "config" in result.output diff --git a/packages/ref/tests/unit/test_config.py b/packages/ref/tests/unit/test_config.py new file mode 100644 index 0000000..81ec7b4 --- /dev/null +++ b/packages/ref/tests/unit/test_config.py @@ -0,0 +1,97 @@ +from pathlib import Path + +import cattrs +import pytest +from ref.config import Config, Paths + + +class TestConfig: + def test_load_missing(self, tmp_path, monkeypatch): + monkeypatch.setenv("REF_CONFIGURATION", str(tmp_path / "ref")) + + # The configuration file doesn't exist + # so it should default to some sane defaults + assert not (tmp_path / "ref.toml").exists() + + loaded = Config.load(Path("ref.toml")) + + assert loaded.paths.data == tmp_path / "ref" / "data" + assert loaded.paths.db == tmp_path / "ref" / "db" + + # The results aren't serialised back to disk + assert not (tmp_path / "ref.toml").exists() + assert loaded._raw is None + assert loaded._config_file == Path("ref.toml") + + def test_default(self, config): + config.paths.data = "data" + config.save() + + # The default location is overridden in the config fixture + loaded = Config.default() + assert loaded.paths.data == Path("data") + + def test_load(self, config, tmp_path): + res = config.dump(defaults=True) + + with open(tmp_path / "ref.toml", "w") as fh: + fh.write(res.as_string()) + + loaded = Config.load(tmp_path / "ref.toml") + + assert config.dumps() == loaded.dumps() + + def test_load_extra_keys(self, tmp_path): + content = """[paths] +data = "data" +extra = "extra" + +[db] +filename = "sqlite://ref.db" +""" + + with open(tmp_path / "ref.toml", "w") as fh: + fh.write(content) + + # cattrs exceptions are a bit ugly, but you get an exception like this: + # + # | cattrs.errors.ClassValidationError: While structuring Config (1 sub-exception) + # +-+---------------- 1 ---------------- + # | Exception Group Traceback (most recent call last): + # | File "", line 6, in structure_Config + # | res['paths'] = __c_structure_paths(o['paths'], __c_type_paths) + # | File "", line 31, in structure_Paths + # | if errors: raise __c_cve('While structuring ' + 'Paths', errors, __cl) + # | cattrs.errors.ClassValidationError: While structuring Paths (1 sub-exception) + # | Structuring class Config @ attribute paths + # +-+---------------- 1 ---------------- + # | cattrs.errors.ForbiddenExtraKeysError: Extra fields in constructor for Paths: extra + + with pytest.raises(cattrs.errors.ClassValidationError): + Config.load(tmp_path / "ref.toml") + + def test_save(self, tmp_path): + config = Config(paths=Paths(data=Path("data"))) + + with pytest.raises(ValueError): + # The configuration file hasn't been set as it was created directly + config.save() + + config.save(tmp_path / "ref.toml") + + assert (tmp_path / "ref.toml").exists() + + def test_defaults(self, monkeypatch): + monkeypatch.setenv("REF_CONFIGURATION", "test") + + cfg = Config.load(Path("test.toml")) + + with_defaults = cfg.dump(defaults=True) + + without_defaults = cfg.dump(defaults=False) + + assert without_defaults == {} + assert with_defaults == { + "paths": {"data": "test/data", "db": "test/db", "log": "test/log", "tmp": "test/tmp"}, + "db": {"filename": "sqlite://ref.db"}, + } diff --git a/pyproject.toml b/pyproject.toml index ae5b0e6..dabebb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,7 @@ addopts = [ ] filterwarnings = [ 'ignore:Deprecated call to `pkg_resources.:DeprecationWarning', + 'ignore:pkg_resources is deprecated as an API:DeprecationWarning', ] diff --git a/stubs/environs/__init__.pyi b/stubs/environs/__init__.pyi new file mode 100644 index 0000000..9f4ae8c --- /dev/null +++ b/stubs/environs/__init__.pyi @@ -0,0 +1,12 @@ +from pathlib import Path + +class Env: + def __init__(self, expand_vars: bool) -> None: ... + def path(self, name: str) -> Path: ... + def read_env( + self, + path: str | None = None, + recurse: bool = True, + verbose: bool = False, + override: bool = False, + ) -> bool: ... diff --git a/uv.lock b/uv.lock index 5a99ebb..2f3f2f5 100644 --- a/uv.lock +++ b/uv.lock @@ -611,6 +611,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, ] +[[package]] +name = "environs" +version = "11.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "marshmallow" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/13/3d448cfbed9f1baff5765f49434cd849501351f14fd3f09f0f2e9bd35322/environs-11.0.0.tar.gz", hash = "sha256:069727a8f73d8ba8d033d3cd95c0da231d44f38f1da773bf076cef168d312ee8", size = 25787 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/30/ef8a3022e6cdcedfd7ba03ca88ab29e30334f8e958cdbf5ce120912397e8/environs-11.0.0-py3-none-any.whl", hash = "sha256:e0bcfd41c718c07a7db422f9109e490746450da38793fe4ee197f397b9343435", size = 12216 }, +] + [[package]] name = "esgpull" version = "0.7.3" @@ -1280,6 +1293,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/87/4c364e0f109eea2402079abecbe33fef4f347b551a11423d1f4e187ea497/MarkupSafe-3.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:730d86af59e0e43ce277bb83970530dd223bf7f2a838e086b50affa6ec5f9295", size = 15741 }, ] +[[package]] +name = "marshmallow" +version = "3.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/30/14d8609f65c8aeddddd3181c06d2c9582da6278f063b27c910bbf9903441/marshmallow-3.23.1.tar.gz", hash = "sha256:3a8dfda6edd8dcdbf216c0ede1d1e78d230a6dc9c5a088f58c4083b974a0d468", size = 177488 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/a7/a78ff54e67ef92a3d12126b98eb98ab8abab3de4a8c46d240c87e514d6bb/marshmallow-3.23.1-py3-none-any.whl", hash = "sha256:fece2eb2c941180ea1b7fcbd4a83c51bfdd50093fdd3ad2585ee5e1df2508491", size = 49488 }, +] + [[package]] name = "matplotlib-inline" version = "0.1.7" @@ -1925,6 +1950,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, ] +[[package]] +name = "python-dotenv" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/57/e84d88dfe0aec03b7a2d4327012c1627ab5f03652216c63d49846d7a6c58/python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca", size = 39115 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", size = 19863 }, +] + [[package]] name = "python-json-logger" version = "2.0.7" @@ -2098,11 +2132,21 @@ name = "ref" version = "0.1.0" source = { editable = "packages/ref" } dependencies = [ + { name = "attrs" }, + { name = "cattrs" }, + { name = "environs" }, + { name = "tomlkit" }, { name = "typer" }, ] [package.metadata] -requires-dist = [{ name = "typer", specifier = ">=0.12.5" }] +requires-dist = [ + { name = "attrs", specifier = ">=24.2.0" }, + { name = "cattrs", specifier = ">=24.1.2" }, + { name = "environs", specifier = ">=11.0.0" }, + { name = "tomlkit", specifier = ">=0.13.2" }, + { name = "typer", specifier = ">=0.12.5" }, +] [[package]] name = "ref-core"