diff --git a/docs/source/helpers/privex.helpers.common.rst b/docs/source/helpers/privex.helpers.common.rst index b30511b..00648fe 100644 --- a/docs/source/helpers/privex.helpers.common.rst +++ b/docs/source/helpers/privex.helpers.common.rst @@ -45,8 +45,10 @@ Common/General Helpers env_int env_keyval extract_settings + extract_type filter_form get_function_params + get_return_type human_name inject_items io_tail @@ -59,6 +61,7 @@ Common/General Helpers shell_quote stringify tail + typing_to_base diff --git a/privex/helpers/__init__.py b/privex/helpers/__init__.py index 5bb3291..cfb4c19 100644 --- a/privex/helpers/__init__.py +++ b/privex/helpers/__init__.py @@ -148,7 +148,7 @@ def _setup_logging(level=logging.WARNING): log = _setup_logging() name = 'helpers' -VERSION = '2.18.0' +VERSION = '2.19.0' diff --git a/privex/helpers/common.py b/privex/helpers/common.py index 846198a..bc5291d 100644 --- a/privex/helpers/common.py +++ b/privex/helpers/common.py @@ -39,7 +39,7 @@ from typing import Sequence, List, Union, Tuple, Type, Dict, Any, Iterable, Optional, BinaryIO, Generator, Mapping from privex.helpers import settings from privex.helpers.collections import DictObject, OrderedDictObject -from privex.helpers.types import T, K, V, C, USE_ORIG_VAR, STRBYTES, Number, NumberStr +from privex.helpers.types import T, K, V, C, USE_ORIG_VAR, STRBYTES, NumberStr from privex.helpers.exceptions import NestedContextException @@ -262,14 +262,15 @@ def parse_csv(line: str, csvsplit: str = ',') -> List[str]: return [x.strip() for x in line.strip().split(csvsplit)] -def env_csv(env_key: str, env_default = None, csvsplit=',') -> List[str]: +def env_csv(env_key: str, env_default=None, csvsplit=',') -> List[str]: """ Quick n' dirty parsing of simple CSV formatted environment variables, with fallback to user specified ``env_default`` (defaults to None) Example: - >>> os.setenv('EXAMPLE', ' hello , world, test') + >>> import os + >>> os.environ['EXAMPLE'] = ' hello , world, test') >>> env_csv('EXAMPLE', []) ['hello', 'world', 'test'] >>> env_csv('NONEXISTANT', []) @@ -502,6 +503,226 @@ class - ``class MyClass:``, with settings defined as static class attributes) return {**defaults, **set_conf, **kwargs, **merge_conf} +def get_return_type(f: callable) -> Optional[Union[type, object, callable]]: + """ + Extract the return type for a function/method. Note that this only works with functions/methods which have their + return type annotated, e.g. ``def somefunc(x: int) -> float: return x * 2.1`` + + .. Attention:: If you want to extract a function/method return type and have any Generic :mod:`typing` types simplified + down to their native Python base types (important to be able to compare with :func:`.isinstance` etc.), + then you should use :func:`.extract_type` instead (handles raw types, objects, and function pointers) + + + **Example 1** - Extracting a generic return type from a function:: + + >>> def list_wrap(v: T) -> List[T]: + ... return [v] + ... + >>> rt = get_return_type(list_wrap) + typing.List[~T] + >>> rt._name # We can get the string type name via _name + 'List' + >>> l = rt.__args__[0] # We can access the types inside of the [] via .__args__ + ~T + >>> l.__name__ # Get the name of 'l' - the type inside of the [] + 'T' + + **Example 2** - What happens if you use this on a function/method with no return type annotation? + + The answer is: **nothing** - it will simply return ``None`` if the function/method has no return type annotation:: + + >>> def hello(x): + ... return x * 5 + >>> repr(get_return_type(hello)) + 'None' + + + :param callable f: A function/method to extract the return type from + :return return_type: The return type, usually either a :class:`.type` or a :class:`.object` + """ + if f is None: return None + if not inspect.isclass(f) and any([inspect.isfunction(f), inspect.ismethod(f), inspect.iscoroutinefunction(f)]): + sig = inspect.signature(f) + ret = sig.return_annotation + # noinspection PyUnresolvedReferences,PyProtectedMember + if ret is inspect._empty or empty(ret, True): + return None + return ret + return f + + +def typing_to_base(tp, fail=False, return_orig=True, clean_union=True) -> Optional[Union[type, object, callable, tuple, Tuple[type]]]: + """ + Attempt to extract one or more native Python base types from a :mod:`typing` type, including generics such as ``List[str]``, + and combined types such as ``Union[list, str]`` + + >>> typing_to_base(List[str]) + list + >>> typing_to_base(Union[str, Dict[str, list], int]) + (str, dict, int) + >>> typing_to_base(Union[str, Dict[str, list], int], clean_union=False) + (str, typing.Dict[str, list], int) + >>> typing_to_base(str) + str + >>> typing_to_base(str, fail=True) + TypeError: Failed to extract base type for type object: + >>> repr(typing_to_base(str, return_orig=False)) + 'None' + + :param tp: The :mod:`typing` type object to extract base/native type(s) from. + :param bool fail: (Default: ``False``) If True, then raises :class:`.TypeError` if ``tp`` doesn't appear to be a :mod:`typing` type. + :param bool return_orig: (Default: ``True``) If True, returns ``tp`` as-is if it's not a typing type. When ``False``, + non- :mod:`typing` types will cause ``None`` to be returned. + :param bool clean_union: (Default: ``True``) If True, :class:`typing.Union`'s will have each type + converted/validated into a normal type using :func:`.extract_type` + :return type_res: Either a :class:`.type` base type, a :class:`.tuple` of types, a :mod:`typing` type object, or something else + depending on what type ``tp`` was. + """ + # We can't use isinstance() with Union generic objects, so we have to identify them by checking their repr string. + if repr(tp).startswith('typing.Union['): + # For Union's (including Optional[]), we iterate over the object's ``__args__`` which contains the Union's types, + # and pass each type through extract_type to cleanup any ``typing`` generics such as ``List[str]`` back into + # their native type (e.g. ``str`` for ``List[str]``) + ntypes = [] + # noinspection PyUnresolvedReferences + targs = tp.__args__ + for t in targs: + try: + ntypes.append(extract_type(t) if clean_union else t) + except Exception as e: + log.warning("Error while extracting type for %s (part of %s). Reason: %s - %s", t, repr(tp), type(e), str(e)) + ntypes.append(t) + return tuple(ntypes) + # For Python 3.6, __origin__ contains the typing type without the generic part, while __orig_bases__ is a tuple containing the + # native/base type, and some typing type. + # On 3.7+, __origin__ contains the native/base type, while __orig_bases__ doesn't exist + if hasattr(tp, '__orig_bases__'): return tp.__orig_bases__[0] + + # __origin__ / __extra__ are exposed by :mod:`typing` types, including generics such as Dict[str,str] + # original SO answer: https://stackoverflow.com/a/54241536/2648583 + if hasattr(tp, '__origin__'): return tp.__origin__ + if hasattr(tp, '__extra__'): return tp.__extra__ + if fail: + raise TypeError(f"Failed to extract base type for type object: {repr(tp)}") + if return_orig: + return tp + return None + + +def extract_type(tp: Union[type, callable, object], **kwargs) -> Optional[Union[type, object, callable, tuple, Tuple[type]]]: + """ + Attempt to identify the :class:`.type` of a given value, or for functions/methods - identify their RETURN value type. + + This function can usually detect :mod:`typing` types, including generics such as ``List[str]``, and will attempt to extract + their native Python base type, e.g. :class:`.list`. + + For :class:`typing.Union` based types (including :class:`typing.Optional`), it can extract a tuple of base types, including + from nested :class:`typing.Union`'s - e.g. ``Union[str, list, Union[dict, set], int`` would be simplified down + to ``(str, list, dict, set, int)`` + + .. Attention:: If you want to extract the original return type from a function/method, including generic types such as ``List[str]``, + then you should use :func:`.get_return_type` instead. + + **Example 1** - convert a generic type e.g. ``Dict[str, str]`` into it's native type (e.g. ``dict``):: + + >>> dtype = Dict[str, str] + >>> # noinspection PyTypeHints,PyTypeChecker + >>> isinstance({}, dtype) + TypeError: Subscripted generics cannot be used with class and instance checks + >>> extract_type(dtype) + dict + >>> isinstance({}, extract_type(dtype)) + True + + **Example 2** - extract the return type of a function/method, and if the return type is a generic (e.g. ``List[str]``), automatically + convert it into the native type (e.g. ``list``) for use in comparisons such as :func:`.isinstance`:: + + >>> def list_wrap(v: T) -> List[T]: + ... return [v] + >>> + >>> extract_type(list_wrap) + list + >>> isinstance([1, 2, 3], extract_type(list_wrap)) + True + + **Example 3** - extract the type from an instantiated object, allowing for :func:`.isinstance` comparisons:: + + >>> from privex.helpers import DictObject + >>> db = DictObject(hello='world', lorem='ipsum') + {'hello': 'world', 'lorem': 'ipsum'} + >>> type_db = extract_type(db) + privex.helpers.collections.DictObject + >>> isinstance(db, type_db) + True + >>> isinstance(DictObject(test=123), type_db) + True + + **Example 4** - extract a tuple of types from a :class:`typing.Union` or :class:`typing.Optional` (inc. return types) :: + + >>> def hello(x) -> Optional[str]: + ... return x * 5 + ... + >>> extract_type(hello) + (str, NoneType) + >>> # Even with a Union[] containing a List[], another Union[] (containing a Tuple and set), and a Dict[], + >>> # extract_type is still able to recursively flatten and simplify it down to a tuple of base Python types + >>> extract_type(Union[ + ... List[str], + ... Union[Tuple[str, int, str], set], + ... Dict[int, str] + ... ]) + (list, tuple, set, dict) + + + **Return Types** + + A :class:`.type` will be returned for most calls where ``tp`` is either: + + * Already a native :class:`.type` e.g. :class:`.list` + * A generic type such as ``List[str]`` (which are technically instances of :class:`.object`) + * A function/method with a valid return type annotation, including generic return types + * An instance of a class (an object), where the original type can be easily extracted via ``tp.__class__`` + + If ``tp`` was an :class:`.object` and the type/class couldn't be extracted, then it would be returned in it's original object form. + + If ``tp`` was an unusual function/method which couldn't be detected as one, or issues occurred while extracting the return type, + then ``tp`` may be returned in it's original :class:`.callable` form. + + + :param tp: The type/object/function etc. to extract the most accurate type from + :return type|object|callable ret: A :class:`.type` will be returned for most calls, but may be an :class:`.object` + or :class:`.callable` if there were issues detecting the type. + """ + # If tp is None, there's nothing we can do with it, so return None. + if tp is None: return None + # If 'tp' is a known native type, we don't need to extract anything, just return tp. + if tp in [list, set, tuple, dict, str, bytes, int, float, Decimal]: return tp + is_func = any([inspect.isfunction(tp), inspect.ismethod(tp), inspect.iscoroutinefunction(tp)]) + # Functions count as class instances (instances of object), therefore to narrow down a real class/type instance, + # we have to confirm it's NOT a function/method/coro, NOT a raw class/type, but IS an instance of object. + # if not is_func and not inspect.isclass(tp) and isinstance(tp, object): + if not is_func and isinstance(tp, object): + # Handle extracting base types from generic :mod:`typing` objects, including tuples of types from Union's + tbase = typing_to_base(tp, return_orig=False) + if tbase is not None: # If the result wasn't None, then we know it was a typing type and base type(s) were extracted properly + return tbase + # Before checking __class__, we make sure that tp is an instance by checking isclass(tp) is False + if not inspect.isclass(tp) and hasattr(tp, '__class__'): + return tp.__class__ # If tp isn't a typing type, __class__ (if it exists) should be the "type" of tp + return tp # If all else fails, return tp as-is + + # If is_func matches at this point, we're dealing with a function/method/coroutine and need to extract the return type. + # To prevent an infinite loop, we set _sec_layer when passing the return type to extract_type(), ensuring that we don't + # call extract_type(rt) AGAIN if the return type just so happened to be a function + if is_func and not kwargs.get('_sec_layer'): + # Extract the original return type, then pass it through extract_type again, since if it's a generic type, + # we'll want to extract the native type from it, since generics like ``List[str]`` can't be used with ``isinstance()`` + rt = get_return_type(tp) + return extract_type(rt, _sec_layer=True) + # If all else fails, return tp as-is + return tp + + def dec_round(amount: Decimal, dp: int = 2, rounding=None) -> Decimal: """ Round a Decimal to x decimal places using ``quantize`` (``dp`` must be >= 1 and the default dp is 2) @@ -768,7 +989,7 @@ def shell_quote(*args: str) -> str: return shlex.join(args) if hasattr(shlex, 'join') else " ".join([shlex.quote(a) for a in args]).strip() -def call_sys(proc, *args, write: STRBYTES = None, **kwargs) -> Tuple[bytes, bytes]: +def call_sys(proc, *args, write: STRBYTES = None, **kwargs) -> Union[Tuple[bytes, bytes], Tuple[str, str]]: """ A small wrapper around :class:`subprocess.Popen` which allows executing processes, while optionally piping data (``write``) into the process's stdin, then finally returning the process's output and error results. @@ -1033,9 +1254,9 @@ def almost(compare: NumberStr, *numbers: NumberStr, tolerance: NumberStr = Decim AssertionError - :param Decimal|int|float compare: The base number which all ``numbers`` will be compared against. - :param Decimal|int|float numbers: One or more numbers to compare against ``compare`` - :param Decimal|int|float tolerance: (kwarg only) Amount that each ``numbers`` can be greater/smaller than ``compare`` before + :param Decimal|int|float|str compare: The base number which all ``numbers`` will be compared against. + :param Decimal|int|float|str numbers: One or more numbers to compare against ``compare`` + :param Decimal|int|float|str tolerance: (kwarg only) Amount that each ``numbers`` can be greater/smaller than ``compare`` before returning ``False``. :keyword bool fail: (default: ``False``) If true, will raise :class:`.AssertionError` on failed tolerance check, instead of returning ``False``. (mutually exclusive with ``assert``) @@ -1076,13 +1297,13 @@ def almost(compare: NumberStr, *numbers: NumberStr, tolerance: NumberStr = Decim """Pre-compiled regex for matching catch-all keyword argument parameter names like ``**args``""" T_PARAM = inspect.Parameter """Type alias for :class:`inspect.Parameter`""" -T_PARAM_LIST = Union[Dict[str, T_PARAM], List[T_PARAM], Iterable[T_PARAM]] +T_PARAM_LIST = Union[Dict[str, T_PARAM], Mapping[str, T_PARAM], List[T_PARAM], Iterable[T_PARAM]] """ Type alias for dict's containing strings mapped to :class:`inspect.Parameter`'s, lists of just :class:`inspect.Parameter`'s, and any iterable of :class:`inspect.Parameter` """ -# noinspection PyProtectedMember +# noinspection PyProtectedMember,PyUnresolvedReferences INS_EMPTY = inspect._empty """ Type alias for :class:`inspect.empty` @@ -1134,8 +1355,6 @@ def _filter_params(params: T_PARAM_LIST, ignore_xargs=False, ignore_xkwargs=Fals ignore_defaults = kwargs.pop('ignore_defaults', False) ignore_positional = kwargs.pop('ignore_positional', False) - - _params = params if isinstance(params, (dict, OrderedDict)) or hasattr(params, 'values'): _params = params.values() diff --git a/tests/general/test_general.py b/tests/general/test_general.py index 4ec4eeb..d1a4a72 100644 --- a/tests/general/test_general.py +++ b/tests/general/test_general.py @@ -26,7 +26,7 @@ from decimal import Decimal from os import path, makedirs from tempfile import TemporaryDirectory, NamedTemporaryFile, mkstemp -from typing import Union, Tuple, List, TextIO, BinaryIO +from typing import Dict, Union, Tuple, List, TextIO, BinaryIO from privex import helpers from tests import PrivexBaseCase import logging @@ -484,3 +484,97 @@ def test_two_numbers_test_kwarg(self): with self.assertRaises(AssertionError): helpers.almost('5', '5.3', test=True) + +def hello_basic_str(x) -> str: return f'{x} world' +def hello_basic_int(x) -> int: return 1234 + x +def hello_generic(x) -> List[str]: return [x] +def hello_generic2(x) -> Union[int, str]: return x +def hello_generic3(x) -> Union[int, Union[dict, set], str]: return x + + +class TestGeneralTypeExtract(PrivexBaseCase): + """ + Test cases for :func:`.get_return_type`, :func:`.typing_to_base` and :func:`.extract_type` + """ + def test_get_return_type_basic(self): + """Test :func:`.get_return_type` with functions returning basic python types""" + self.assertIs(helpers.get_return_type(hello_basic_str), str) + self.assertIs(helpers.get_return_type(hello_basic_int), int) + + def test_get_return_type_generic(self): + """Test :func:`.get_return_type` with functions returning generic :mod:`typing` types""" + g1, g2 = helpers.get_return_type(hello_generic), helpers.get_return_type(hello_generic2) + if hasattr(g1, '_name'): # Python 3.6 doesn't have _name, so only test it if it exists. + self.assertEqual(g1._name, 'List') + self.assertEqual('typing.List[str]', repr(g1)) + self.assertEqual('typing.Union[int, str]', repr(g2)) + self.assertTupleEqual(g2.__args__, (int, str)) + + def test_typing_to_base_basic(self): + """Test :func:`.typing_to_base` with basic :mod:`typing` types such as :class:`typing.List`""" + self.assertIs(helpers.typing_to_base(List[str]), list) + self.assertIs(helpers.typing_to_base(Dict[str, int]), dict) + + def test_typing_to_base_union(self): + """Test :func:`.typing_to_base` with more complex :class:`typing.Union` types""" + self.assertTupleEqual(helpers.typing_to_base(Union[str, List[dict], int]), (str, list, int)) + self.assertTupleEqual(helpers.typing_to_base( + Union[List[str], Union[Tuple[str, int, str], set], Dict[int, str]] + ), (list, tuple, set, dict)) + + def test_typing_to_base_invalid(self): + """Test :func:`.typing_to_base` with non :mod:`typing` types such as :class:`str`""" + self.assertIs(helpers.typing_to_base(str), str) + self.assertIsNone(helpers.typing_to_base(str, return_orig=False)) + with self.assertRaises(TypeError): + helpers.typing_to_base(str, fail=True) + + def test_typing_to_base_union_no_clean(self): + """Test :func:`.typing_to_base` with clean_union disabled and enabled with a :class:`typing.Union`""" + b = helpers.typing_to_base(Union[str, List[dict], int], clean_union=False) + self.assertIs(b[0], str) + self.assertEqual(repr(b[1]), 'typing.List[dict]') + self.assertIs(b[2], int) + b = helpers.typing_to_base(Union[str, List[dict], int], clean_union=True) + self.assertIs(b[1], list) + + def test_extract_type_type(self): + """Test :func:`.extract_type` with simple types such as :class:`str` and :class:`.PrivexBaseCase`""" + self.assertIs(helpers.extract_type(str), str) + self.assertIs(helpers.extract_type(int), int) + self.assertIs(helpers.extract_type(dict), dict) + self.assertIs(helpers.extract_type(PrivexBaseCase), PrivexBaseCase) + + def test_extract_type_instance(self): + """Test :func:`.extract_type` with object instances""" + self.assertIs(helpers.extract_type(dict(hello='world')), dict) + self.assertIs(helpers.extract_type(list('world')), list) + self.assertIs(helpers.extract_type(set('world')), set) + + class Hello: + def __init__(self): pass + + self.assertIs(helpers.extract_type(Hello()), Hello) + self.assertIsInstance(Hello(), helpers.extract_type(Hello())) + + def test_extract_type_generic(self): + """Test :func:`.extract_type` with generic :mod:`typing` types such as :class:`typing.Tuple`""" + self.assertIs(helpers.extract_type(List[str]), list) + self.assertIs(helpers.extract_type(Tuple[int, str]), tuple) + self.assertIs(helpers.extract_type(Dict[str, int]), dict) + self.assertTupleEqual(helpers.extract_type(Union[str, int]), (str, int)) + + def test_extract_type_func_basic(self): + """Test :func:`.extract_type` with functions that return basic types""" + self.assertIs(helpers.extract_type(hello_basic_str), str) + self.assertIs(helpers.extract_type(hello_basic_int), int) + + def test_extract_type_func_generic(self): + """Test :func:`.extract_type` with functions that return :mod:`typing` generic types and unions""" + self.assertIs(helpers.extract_type(hello_generic), list) + self.assertTupleEqual(helpers.extract_type(hello_generic2), (int, str)) + self.assertTupleEqual(helpers.extract_type(hello_generic3), (int, dict, set, str)) + + + +