diff --git a/README.md b/README.md index 4e1a194..8fef71e 100644 --- a/README.md +++ b/README.md @@ -66,10 +66,9 @@ Then configuration is built from: - `PYTHON_ALPHACONF` environment variable may contain a path to load - configuration files from configuration directories (using application name) - environment variables based on key prefixes, - except "BASE" and "PYTHON"; + except "BASE" and "PYTHON"; \ if you have a configuration key "abc", all environment variables starting - with "ABC_" will be loaded where keys are converted to lower case and "_" - to ".": "ABC_HELLO=a" would set "abc.hello=a" + with "ABC_" will be loaded, for example "ABC_HELLO=a" would set "abc.hello=a" - key-values from the program arguments Finally, the configuration is fully resolved and logging is configured. @@ -104,10 +103,11 @@ class MyConf(pydantic.BaseModel): def build(self): # use as a factory pattern to create more complex objects + # for example, a connection to the database return self.value * 2 # setup the configuration -alphaconf.setup_configuration(MyConf, path='a') +alphaconf.setup_configuration(MyConf, prefix='a') # read the value alphaconf.get('a', MyConf) v = alphaconf.get(MyConf) # because it's registered as a type @@ -122,6 +122,29 @@ You can read values or passwords from files, by using the template or, more securely, read the file in the code `alphaconf.get('secret_file', Path).read_text().strip()`. +### Inject parameters + +We can inject default values to functions from the configuration. +Either one by one, where we can map a factory function or a configuration key. +Or inject all automatically base on the parameter name. + +```python +from alphaconf.inject import inject, inject_auto + +@inject('name', 'application.name') +@inject_auto(ignore={'name'}) +def main(name: str, example=None): + pass + +# similar to +def main(name: str=None, example=None): + if name is None: + name = alphaconf.get('application.name', str) + if example is None: + example = alphaconf.get('example', default=example) + ... +``` + ### Invoke integration Just add the lines below to parameterize invoke. @@ -136,9 +159,8 @@ alphaconf.invoke.run(__name__, ns) ``` ## Way to 1.0 -- Run function `@alphaconf.inject` -- Run a specific function `alphaconf.cli.run_module()`: - find functions and parse their args +- Run a specific function `alphaconf my.module.main`: + find functions and inject args - Install completions for bash `alphaconf --install-autocompletion` [OmegaConf]: https://omegaconf.readthedocs.io/ diff --git a/alphaconf/inject.py b/alphaconf/inject.py new file mode 100644 index 0000000..ffd8620 --- /dev/null +++ b/alphaconf/inject.py @@ -0,0 +1,92 @@ +import functools +import inspect +from typing import Any, Callable, Dict, Optional, Union + +import alphaconf + +from .internal.type_resolvers import type_from_annotation + +__all__ = ["inject", "inject_auto"] + + +class ParamDefaultsFunction: + """Function wrapper that injects default parameters""" + + _arg_factory: Dict[str, Callable[[], Any]] + + def __init__(self, func: Callable): + self.func = func + self.signature = inspect.signature(func) + self._arg_factory = {} + + def bind(self, name: str, factory: Callable[[], Any]): + self._arg_factory[name] = factory + + def __call__(self, *a, **kw): + args = self.signature.bind_partial(*a, **kw).arguments + kw.update( + {name: factory() for name, factory in self._arg_factory.items() if name not in args} + ) + return self.func(*a, **kw) + + @staticmethod + def wrap(func) -> "ParamDefaultsFunction": + if isinstance(func, ParamDefaultsFunction): + return func + return functools.wraps(func)(ParamDefaultsFunction(func)) + + +def getter( + key: str, ktype: Optional[type] = None, *, param: Optional[inspect.Parameter] = None +) -> Callable[[], Any]: + """Factory function that calls alphaconf.get + + The parameter from the signature can be given to extract the type to cast to + and whether the configuration value is optional. + + :param key: The key using in alphaconf.get + :param ktype: Type to cast to + :param param: The parameter object from the signature + """ + if ktype is None and param and (ptype := param.annotation) is not param.empty: + ktype = next(type_from_annotation(ptype), None) + if param is not None and param.default is not param.empty: + xparam = param + return ( + lambda: xparam.default + if (value := alphaconf.get(key, ktype, default=None)) is None + and xparam.default is not xparam.empty + else value + ) + return lambda: alphaconf.get(key, ktype) + + +def inject(name: str, factory: Union[None, str, Callable[[], Any]]): + """Inject an argument to a function from a factory or alphaconf""" + + def do_inject(func): + f = ParamDefaultsFunction.wrap(func) + if isinstance(factory, str) or factory is None: + b = getter(factory or name, param=f.signature.parameters[name]) + else: + b = factory + f.bind(name, b) + return f + + return do_inject + + +def inject_auto(*, prefix: str = "", ignore: set = set()): + """Inject automatically all paramters""" + if prefix and not prefix.endswith("."): + prefix += "." + + def do_inject(func): + f = ParamDefaultsFunction.wrap(func) + for name, param in f.signature.parameters.items(): + if name in ignore: + continue + f.bind(name, getter(prefix + name, param=param)) + return f + + return do_inject diff --git a/alphaconf/internal/configuration.py b/alphaconf/internal/configuration.py index 8da1d1e..083effd 100644 --- a/alphaconf/internal/configuration.py +++ b/alphaconf/internal/configuration.py @@ -1,6 +1,5 @@ import copy import os -import typing import warnings from enum import Enum from typing import ( @@ -19,7 +18,7 @@ from omegaconf import Container, DictConfig, OmegaConf -from .type_resolvers import convert_to_type, pydantic +from .type_resolvers import convert_to_type, pydantic, type_from_annotation T = TypeVar('T') @@ -92,14 +91,14 @@ def get(self, key: Union[str, Type], type=None, *, default=raise_on_missing): ) if value is raise_on_missing: if default is raise_on_missing: - raise ValueError(f"No value for: {key}") + raise KeyError(f"No value for: {key}") return default # check the returned type and convert when necessary if type is not None and isinstance(value, type): return value if isinstance(value, Container): value = OmegaConf.to_object(value) - if type is not None and default is not None: + if type is not None and value is not default: value = convert_to_type(value, type) return value @@ -110,12 +109,12 @@ def __get_type(self, key: Type, *, default=raise_on_missing): key_str = self.__type_path.get(key) if key_str is None: if default is raise_on_missing: - raise ValueError(f"Key not found for type {key}") + raise KeyError(f"Key not found for type {key}") return default try: value = self.get(key_str, key) self.__type_value = value - except ValueError: + except KeyError: if default is raise_on_missing: raise value = default @@ -130,7 +129,7 @@ def setup_configuration( conf: Union[DictConfig, dict, Any], helpers: Dict[str, str] = {}, *, - path: str = "", + prefix: str = "", ): """Add a default configuration @@ -146,10 +145,10 @@ def setup_configuration( conf_type = None if conf_type: # if already registered, set path to None - self.__type_path[conf_type] = None if conf_type in self.__type_path else path + self.__type_path[conf_type] = None if conf_type in self.__type_path else prefix self.__type_value.pop(conf_type, None) - if path and not path.endswith('.'): - path += "." + if prefix and not prefix.endswith('.'): + prefix += "." if isinstance(conf, str): warnings.warn("provide a dict directly", DeprecationWarning) created_config = OmegaConf.create(conf) @@ -157,16 +156,16 @@ def setup_configuration( raise ValueError("The config is not a dict") conf = created_config if isinstance(conf, DictConfig): - config = self.__prepare_dictconfig(conf, path=path) + config = self.__prepare_dictconfig(conf, path=prefix) else: - created_config = self.__prepare_config(conf, path=path) + created_config = self.__prepare_config(conf, path=prefix) if not isinstance(created_config, DictConfig): - raise ValueError("Failed to convert to a DictConfig") + raise TypeError("Failed to convert to a DictConfig") config = created_config - # add path and merge - if path: - config = self.__add_path(config, path.rstrip(".")) - helpers = {path + k: v for k, v in helpers.items()} + # add prefix and merge + if prefix: + config = self.__add_prefix(config, prefix.rstrip(".")) + helpers = {prefix + k: v for k, v in helpers.items()} self._merge([config]) # helpers self.helpers.update(**helpers) @@ -221,12 +220,12 @@ def __prepare_dictconfig( sub_configs = [] for k, v in obj.items_ex(resolve=False): if not isinstance(k, str): - raise ValueError("Expecting only str instances in dict") + raise TypeError("Expecting only str instances in dict") if recursive: v = self.__prepare_config(v, path + k + ".") if '.' in k: obj.pop(k) - sub_configs.append(self.__add_path(v, k)) + sub_configs.append(self.__add_prefix(v, k)) if sub_configs: obj = cast(DictConfig, OmegaConf.unsafe_merge(obj, *sub_configs)) return obj @@ -252,9 +251,6 @@ def __prepare_pydantic(self, obj, path): # pydantic instance, prepare helpers self.__prepare_pydantic(type(obj), path) return obj.model_dump(mode="json") - # parse typing recursively for documentation - for t in typing.get_args(obj): - self.__prepare_pydantic(t, path) # check if not a type if not isinstance(obj, type): return obj @@ -279,13 +275,14 @@ def __prepare_pydantic(self, obj, path): from alphaconf import SECRET_MASKS SECRET_MASKS.append(lambda s: s == path) - elif check_type and field.annotation: - self.__prepare_pydantic(field.annotation, path + k + ".") + elif check_type: + for ftype in type_from_annotation(field.annotation): + self.__prepare_pydantic(ftype, path + k + ".") return defaults return None @staticmethod - def __add_path(config: Any, path: str) -> DictConfig: - for part in reversed(path.split(".")): + def __add_prefix(config: Any, prefix: str) -> DictConfig: + for part in reversed(prefix.split(".")): config = OmegaConf.create({part: config}) return config diff --git a/alphaconf/internal/type_resolvers.py b/alphaconf/internal/type_resolvers.py index be49a47..d63e052 100644 --- a/alphaconf/internal/type_resolvers.py +++ b/alphaconf/internal/type_resolvers.py @@ -1,4 +1,5 @@ import datetime +import typing from pathlib import Path from omegaconf import OmegaConf @@ -16,11 +17,7 @@ """ -def read_text(value): - return Path(value).expanduser().read_text() - - -def parse_bool(value) -> bool: +def _parse_bool(value) -> bool: if isinstance(value, str): value = value.strip().lower() if value in ('no', 'false', 'n', 'f', 'off', 'none', 'null', 'undefined', '0'): @@ -29,14 +26,14 @@ def parse_bool(value) -> bool: TYPE_CONVERTER = { - bool: parse_bool, + bool: _parse_bool, datetime.datetime: datetime.datetime.fromisoformat, datetime.date: lambda s: datetime.datetime.strptime(s, '%Y-%m-%d').date(), datetime.time: datetime.time.fromisoformat, - Path: lambda s: Path(s).expanduser(), + Path: lambda s: Path(str(s)).expanduser(), str: lambda v: str(v), - 'read_text': read_text, - 'read_strip': lambda s: read_text(s).strip(), + 'read_text': lambda s: Path(s).expanduser().read_text(), + 'read_strip': lambda s: Path(s).expanduser().read_text().strip(), 'read_bytes': lambda s: Path(s).expanduser().read_bytes(), } @@ -65,3 +62,12 @@ def convert_to_type(value, type): if pydantic: return pydantic.TypeAdapter(type).validate_python(value) return type(value) + + +def type_from_annotation(annotation) -> typing.Generator[type, None, None]: + """Given an annotation (optional), figure out the types""" + if isinstance(annotation, type) and annotation is not type(None): + yield annotation + else: + for t in typing.get_args(annotation): + yield from type_from_annotation(t) diff --git a/example-typed.py b/example-typed.py index ece0717..fc2b1bd 100755 --- a/example-typed.py +++ b/example-typed.py @@ -20,7 +20,7 @@ class MyConfiguration(BaseModel): connection: Optional[Conn] = None -alphaconf.setup_configuration(MyConfiguration, path="c") +alphaconf.setup_configuration(MyConfiguration, prefix="c") def main(): diff --git a/pyproject.toml b/pyproject.toml index af22d06..b9a7e56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,13 +23,13 @@ classifiers = [ ] dependencies = [ "omegaconf>=2", - "pydantic>=2", ] [project.optional-dependencies] color = ["colorama"] dotenv = ["python-dotenv"] invoke = ["invoke"] +pydantic = ["pydantic>=2"] toml = ["toml"] [project.urls] diff --git a/tests/test_alphaconf.py b/tests/test_alphaconf.py index 3544057..add95b7 100644 --- a/tests/test_alphaconf.py +++ b/tests/test_alphaconf.py @@ -90,7 +90,7 @@ def test_setup_configuration(): def test_setup_configuration_invalid(): - with pytest.raises(ValueError): + with pytest.raises(TypeError): # invalid configuration (must be non-empty) alphaconf.setup_configuration(None) @@ -132,8 +132,8 @@ def test_app_environ(application): ) application.setup_configuration(load_dotenv=False, env_prefixes=True) config = application.configuration - with pytest.raises(ValueError): - # XXX should not be loaded + with pytest.raises(KeyError): + # prefix with underscore only should be loaded config.get('xxx') assert config.get('testmyenv.x') == 'overwrite' assert config.get('testmyenv.y') == 'new' diff --git a/tests/test_configuration.py b/tests/test_configuration.py index b9386f0..05fa83b 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -48,6 +48,7 @@ def test_cast(config): # cast bool into int assert config.get('b') is True assert config.get('b', int) == 1 + assert config.get('b', int, default=None) == 1 # cast Path assert isinstance(config.get('home', Path), Path) @@ -71,7 +72,7 @@ def test_select_empty(config): def test_select_required(config): assert config.get('z', default=None) is None - with pytest.raises(ValueError): + with pytest.raises(KeyError): print(config.get('z')) assert config.get('z', default='a') == 'a' @@ -80,7 +81,7 @@ def test_select_required_incomplete(config_req): # when we have a default, return it assert config_req.get('req', default='def') == 'def' # when required, raise missing - with pytest.raises(ValueError): + with pytest.raises(KeyError): print(config_req.get('req')) @@ -132,5 +133,5 @@ def test_config_setup_dots(config): def test_config_setup_path(config): - config.setup_configuration({'test': 954}, path='a.b') + config.setup_configuration({'test': 954}, prefix='a.b') assert config.get('a.b.test') == 954 diff --git a/tests/test_configuration_typed.py b/tests/test_configuration_typed.py index 3f1808d..5b55097 100644 --- a/tests/test_configuration_typed.py +++ b/tests/test_configuration_typed.py @@ -95,6 +95,6 @@ def test_set_person(config_typed): def test_set_person_type(config_typed): - config_typed.setup_configuration(Person(first_name='A', last_name='T'), path='x_person') + config_typed.setup_configuration(Person(first_name='A', last_name='T'), prefix='x_person') person = config_typed.get(Person) assert person.full_name == 'A T' diff --git a/tests/test_inject.py b/tests/test_inject.py new file mode 100644 index 0000000..05a73f0 --- /dev/null +++ b/tests/test_inject.py @@ -0,0 +1,81 @@ +import string +from typing import Optional + +import pytest + +import alphaconf +import alphaconf.inject as inj + + +@pytest.fixture(scope="function") +def c(): + alphaconf.setup_configuration(dict(zip(string.ascii_letters, range(1, 11)))) + alphaconf.set_application(app := alphaconf.Application()) + return app.configuration + + +def mytuple(a: int, b=1, *, c, d=1, zz: int = 1): + return (a, b, c, d, zz) + + +@pytest.fixture(scope="function") +def mytupledef(): + return inj.inject("a", lambda: 1)(inj.inject("c", lambda: 1)(mytuple)) + + +def test_inject(c, mytupledef): + assert mytuple(0, c=2) == (0, 1, 2, 1, 1) + assert mytupledef() == (1, 1, 1, 1, 1) + assert inj.inject("c", lambda: 5)(mytuple)(0) == (0, 1, 5, 1, 1) + assert inj.inject("b", lambda: 5)(mytupledef)() == (1, 5, 1, 1, 1) + + +def test_inject_name(c, mytupledef): + assert inj.inject('a', 'g')(mytuple)(c=0) == (7, 1, 0, 1, 1) + + +def test_inject_auto_lambda(c): + assert inj.inject_auto()(lambda a: a + 1)() == 2 + assert inj.inject_auto()(lambda c: c + 1)() == 4 + + +def test_inject_auto(c): + assert inj.inject_auto()(mytuple)() == (1, 2, 3, 4, 1) + + +def test_inject_auto_ignore(c): + assert inj.inject_auto(ignore={'b'})(mytuple)() == (1, 1, 3, 4, 1) + + +def test_inject_auto_missing(): + with pytest.raises(KeyError, match=": a"): + inj.inject_auto()(mytuple)() + + +def test_inject_auto_prefix(): + def f1(name): + return name + + alphaconf.setup_configuration({"mytest.name": "ok"}) + assert inj.inject_auto(prefix="mytest")(f1)() == "ok" + + +def test_inject_type_def(mytupledef): + with pytest.raises(KeyError): + inj.inject("a", "nothing")(mytupledef)() + + +def test_inject_type_cast(c): + def f1(zz: str): + return zz + + def f2(zz: str = "ok"): + return zz + + def f3(zz: Optional[str] = None): + return zz + + assert inj.inject('zz', 'g')(f1)() == "7" + assert inj.inject('zz', 'nothing')(f2)() == "ok" + assert inj.inject('zz', 'nothing')(f3)() is None + assert inj.inject('zz', 'g')(f3)() == "7"