From 19a0164ec30f2b48069abecdef2454edfc574a36 Mon Sep 17 00:00:00 2001 From: "Chris (Someguy123)" Date: Fri, 19 Jul 2019 13:48:40 +0100 Subject: [PATCH] v1.1.4 - Added `env_bool` function, csv/kval functions now allow separator to be overriden. **New functions / code improvements** - Added `env_bool` function to `common.py` for loading an environment var as a boolean - Added `csvsplit` parameter to parse/env_csv and keyval, to allow changing the item separator from `,` - Added `valsplit` parameter to parse/env_keyval, to allow customising the separator between key's and value's from `:` **Documentation** - Added PyDoc param's / return's for various functions, and fleshed out some others - Wrapped various docstring values such as ``True`` and ``0`` with backticks so they display better - Various small formatting improvements to existing docstrings - Added `PrivexBaseCase` and `env_bool` to the docs **Unit Testing** - Refactored various attributes in `test.py` into the base class `PrivexBaseClass` - Added example to the PyDoc in `tests.py` showing how to run tests with `pytest` - Wrote new unit tests: - `test_kval_custom_clean` - Validates that `parse_keyval` works properly with custom `valsplit`/`csvsplit` - `test_kval_custom_spaced` - Validates that `parse_keyval` works properly with space padded values and custom `valsplit`/`csvsplit` - `test_env_nonexist_bool`, `test_env_bool_true`, `test_env_bool_false` - Validate that the new `env_bool` function returns the correct values. --- .../common/privex.helpers.common.env_bool.rst | 6 + docs/source/helpers/privex.helpers.common.rst | 1 + .../stubs/tests/tests.PrivexBaseCase.rst | 99 ++++++++++++++++ docs/source/helpers/tests.rst | 1 + privex/helpers/common.py | 101 +++++++++++----- setup.py | 2 +- tests.py | 110 +++++++++++++++--- 7 files changed, 273 insertions(+), 47 deletions(-) create mode 100644 docs/source/helpers/common/privex.helpers.common.env_bool.rst create mode 100644 docs/source/helpers/stubs/tests/tests.PrivexBaseCase.rst diff --git a/docs/source/helpers/common/privex.helpers.common.env_bool.rst b/docs/source/helpers/common/privex.helpers.common.env_bool.rst new file mode 100644 index 0000000..9c8cd8c --- /dev/null +++ b/docs/source/helpers/common/privex.helpers.common.env_bool.rst @@ -0,0 +1,6 @@ +privex.helpers.common.env\_bool +=============================== + +.. currentmodule:: privex.helpers.common + +.. autofunction:: env_bool \ No newline at end of file diff --git a/docs/source/helpers/privex.helpers.common.rst b/docs/source/helpers/privex.helpers.common.rst index b978959..df7c78a 100644 --- a/docs/source/helpers/privex.helpers.common.rst +++ b/docs/source/helpers/privex.helpers.common.rst @@ -13,6 +13,7 @@ privex.helpers.common empty env_csv env_keyval + env_bool is_false is_true parse_csv diff --git a/docs/source/helpers/stubs/tests/tests.PrivexBaseCase.rst b/docs/source/helpers/stubs/tests/tests.PrivexBaseCase.rst new file mode 100644 index 0000000..5c84c7d --- /dev/null +++ b/docs/source/helpers/stubs/tests/tests.PrivexBaseCase.rst @@ -0,0 +1,99 @@ +tests.PrivexBaseCase +==================== + +.. currentmodule:: tests + +.. autoclass:: PrivexBaseCase + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~PrivexBaseCase.__init__ + ~PrivexBaseCase.addCleanup + ~PrivexBaseCase.addTypeEqualityFunc + ~PrivexBaseCase.assertAlmostEqual + ~PrivexBaseCase.assertAlmostEquals + ~PrivexBaseCase.assertCountEqual + ~PrivexBaseCase.assertDictContainsSubset + ~PrivexBaseCase.assertDictEqual + ~PrivexBaseCase.assertEqual + ~PrivexBaseCase.assertEquals + ~PrivexBaseCase.assertFalse + ~PrivexBaseCase.assertGreater + ~PrivexBaseCase.assertGreaterEqual + ~PrivexBaseCase.assertIn + ~PrivexBaseCase.assertIs + ~PrivexBaseCase.assertIsInstance + ~PrivexBaseCase.assertIsNone + ~PrivexBaseCase.assertIsNot + ~PrivexBaseCase.assertIsNotNone + ~PrivexBaseCase.assertLess + ~PrivexBaseCase.assertLessEqual + ~PrivexBaseCase.assertListEqual + ~PrivexBaseCase.assertLogs + ~PrivexBaseCase.assertMultiLineEqual + ~PrivexBaseCase.assertNotAlmostEqual + ~PrivexBaseCase.assertNotAlmostEquals + ~PrivexBaseCase.assertNotEqual + ~PrivexBaseCase.assertNotEquals + ~PrivexBaseCase.assertNotIn + ~PrivexBaseCase.assertNotIsInstance + ~PrivexBaseCase.assertNotRegex + ~PrivexBaseCase.assertNotRegexpMatches + ~PrivexBaseCase.assertRaises + ~PrivexBaseCase.assertRaisesRegex + ~PrivexBaseCase.assertRaisesRegexp + ~PrivexBaseCase.assertRegex + ~PrivexBaseCase.assertRegexpMatches + ~PrivexBaseCase.assertSequenceEqual + ~PrivexBaseCase.assertSetEqual + ~PrivexBaseCase.assertTrue + ~PrivexBaseCase.assertTupleEqual + ~PrivexBaseCase.assertWarns + ~PrivexBaseCase.assertWarnsRegex + ~PrivexBaseCase.assert_ + ~PrivexBaseCase.countTestCases + ~PrivexBaseCase.debug + ~PrivexBaseCase.defaultTestResult + ~PrivexBaseCase.doCleanups + ~PrivexBaseCase.fail + ~PrivexBaseCase.failIf + ~PrivexBaseCase.failIfAlmostEqual + ~PrivexBaseCase.failIfEqual + ~PrivexBaseCase.failUnless + ~PrivexBaseCase.failUnlessAlmostEqual + ~PrivexBaseCase.failUnlessEqual + ~PrivexBaseCase.failUnlessRaises + ~PrivexBaseCase.id + ~PrivexBaseCase.run + ~PrivexBaseCase.setUp + ~PrivexBaseCase.setUpClass + ~PrivexBaseCase.shortDescription + ~PrivexBaseCase.skipTest + ~PrivexBaseCase.subTest + ~PrivexBaseCase.tearDown + ~PrivexBaseCase.tearDownClass + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~PrivexBaseCase.empty_lst + ~PrivexBaseCase.empty_vals + ~PrivexBaseCase.empty_zero + ~PrivexBaseCase.falsey + ~PrivexBaseCase.falsey_empty + ~PrivexBaseCase.longMessage + ~PrivexBaseCase.maxDiff + ~PrivexBaseCase.truthy + + \ No newline at end of file diff --git a/docs/source/helpers/tests.rst b/docs/source/helpers/tests.rst index d952eae..374584e 100644 --- a/docs/source/helpers/tests.rst +++ b/docs/source/helpers/tests.rst @@ -18,6 +18,7 @@ tests TestBoolHelpers TestIPReverseDNS TestParseHelpers + PrivexBaseCase diff --git a/privex/helpers/common.py b/privex/helpers/common.py index 121cded..b9bb669 100644 --- a/privex/helpers/common.py +++ b/privex/helpers/common.py @@ -59,8 +59,8 @@ def random_str(size:int = 50, chars: Sequence = SAFE_CHARS) -> str: SystemRandom class to provide relatively secure randomness from the OS. (On Linux, uses /dev/urandom) By default, uses the character set :py:attr:`.SAFE_CHARS` which contains letters a-z / A-Z and numbers 2-9 - with commonly misread characters removed (such as 1, l, L, 0 and o). Pass :py:attr:`.ALPHANUM` as `chars` if - you needthe full set of upper/lowercase + numbers. + with commonly misread characters removed (such as ``1``, ``l``, ``L``, ``0`` and ``o``). Pass + :py:attr:`.ALPHANUM` as `chars` if you need the full set of upper/lowercase + numbers. Usage: @@ -86,10 +86,10 @@ def random_str(size:int = 50, chars: Sequence = SAFE_CHARS) -> str: def empty(v, zero: bool = False, itr: bool = False) -> bool: """ - Quickly check if a variable is empty or not. By default only '' and None are checked, use `itr` and `zero` to + Quickly check if a variable is empty or not. By default only '' and None are checked, use ``itr`` and ``zero`` to test for empty iterable's and zeroed variables. - Returns True if a variable is None or '', returns False if variable passes the tests + Returns ``True`` if a variable is ``None`` or ``''``, returns ``False`` if variable passes the tests Example usage: @@ -101,10 +101,10 @@ def empty(v, zero: bool = False, itr: bool = False) -> bool: ... print('Var x is None, blank string, or an empty dict/list/iterable') :param v: The variable to check if it's empty - :param zero: if zero=True, then return True if the variable is int 0 or str '0' - :param itr: if itr=True, then return True if the variable is ``[]``, ``{}``, or is an iterable and has 0 length - :return bool is_blank: True if a variable is blank (``None``, ``''``, ``0``, ``[]`` etc.) - :return bool is_blank: False if a variable has content (or couldn't be checked properly) + :param zero: if ``zero=True``, then return ``True`` if the variable is int ``0`` or str ``'0'`` + :param itr: if ``itr=True``, then return ``True`` if the variable is ``[]``, ``{}``, or is an iterable and has 0 length + :return bool is_blank: ``True`` if a variable is blank (``None``, ``''``, ``0``, ``[]`` etc.) + :return bool is_blank: ``False`` if a variable has content (or couldn't be checked properly) """ _check = [None, ''] @@ -118,8 +118,11 @@ def empty(v, zero: bool = False, itr: bool = False) -> bool: def is_true(v) -> bool: """ - Check if a given bool/str/int value is some form of True: - boolean: True // string: 'true', 'yes', 'y', '1' // integer: 1 + Check if a given bool/str/int value is some form of ``True``: + + * **bool**: ``True`` + * **str**: ``'true'``, ``'yes'``, ``'y'``, ``'1'`` + * **int**: ``1`` (note: strings are automatically .lower()'d) @@ -131,7 +134,7 @@ def is_true(v) -> bool: False :param Any v: The value to check for truthfulness - :return bool is_true: True if the value appears to be truthy, otherwise False. + :return bool is_true: ``True`` if the value appears to be truthy, otherwise ``False``. """ v = v.lower() if type(v) is str else v return v in [True, 'true', 'yes', 'y', '1', 1] @@ -141,9 +144,11 @@ def is_false(v, chk_none: bool = True) -> bool: **Warning:** Unless you specifically need to verify a value is Falsey, it's usually safer to check for truth :py:func:`.is_true` and invert the result, i.e. ``if not is_true(v)`` - Check if a given bool/str/int value is some form of False:: + Check if a given bool/str/int value is some form of ``False``: - boolean: False // string: 'false', 'no', 'n', '0' // integer: 0 + * **bool**: ``False`` + * **str**: ``'false'``, ``'no'``, ``'n'``, ``'0'`` + * **int**: ``0`` If ``chk_none`` is True (default), will also consider the below values to be Falsey:: @@ -159,15 +164,15 @@ def is_false(v, chk_none: bool = True) -> bool: False :param Any v: The value to check for falseyness - :param bool chk_none: If True, treat None/'none'/'null' as Falsey (default True) - :return bool is_False: True if the value appears to be falsey, otherwise False. + :param bool chk_none: If ``True``, treat ``None``/``'none'``/``'null'`` as Falsey (default ``True``) + :return bool is_False: ``True`` if the value appears to be falsey, otherwise ``False``. """ v = v.lower() if type(v) is str else v chk = [False, 'false', 'no', 'n', '0', 0] chk += [None, 'none', 'null', ''] if chk_none else [] return v in chk -def parse_keyval(line: str) -> List[Tuple[str, str]]: +def parse_keyval(line: str, valsplit: str = ':', csvsplit=',') -> List[Tuple[str, str]]: """ Parses a csv with key:value pairs such as:: @@ -182,22 +187,20 @@ def parse_keyval(line: str) -> List[Tuple[str, str]]: ] + By default, uses a colons ``:`` to split the key/value, and commas ``,`` to terminate the end of + each keyval pair. This can be overridden by changing valsplit/csvsplit. + :param str line: A string of key:value pairs separated by commas e.g. ``John Alex:Doe,Jane Sarah:Doe`` + :param str valsplit: A character (or several) used to split the key from the value (default: colon ``:``) + :param str csvsplit: A character (or several) used to terminate each keyval pair (default: comma ``,``) :return List[Tuple[str,str]] parsed_data: A list of (key, value) tuples that can easily be casted to a dict() """ - line = [tuple(a.split(':')) for a in line.split(',')] if line != '' else [] + cs, vs = csvsplit, valsplit + line = [tuple(a.split(vs)) for a in line.split(cs)] if line != '' else [] return [(a.strip(), b.strip()) for a, b in line] -def env_keyval(env_key: str, env_default = None) -> List[Tuple[str, str]]: - """ - Parses ``key:val,key:val`` into a list of tuples [(key,val), (key,val)] - - See :py:meth:`parse_keyval` - """ - d = env(env_key) - return env_default if empty(d) else parse_keyval(d) -def parse_csv(line: str) -> List[str]: +def parse_csv(line: str, csvsplit: str = ',') -> List[str]: """ Quick n' dirty parsing of a simple comma separated line, with automatic whitespace stripping of both the ``line`` itself, and the values within the commas. @@ -206,11 +209,14 @@ def parse_csv(line: str) -> List[str]: >>> parse_csv(' hello , world, test') ['hello', 'world', 'test'] + >>> parse_csv(' world ; test ; example', csvsplit=';') + ['world', 'test', 'example'] + :param str csvsplit: A character (or several) used to terminate each value in the list. Default: comma ``,`` """ - return [x.strip() for x in line.strip().split(',')] + return [x.strip() for x in line.strip().split(csvsplit)] -def env_csv(env_key: str, env_default = None) -> List[str]: +def env_csv(env_key: str, env_default = None, csvsplit=',') -> List[str]: """ Quick n' dirty parsing of simple CSV formatted environment variables, with fallback to user specified ``env_default`` (defaults to None) @@ -224,11 +230,46 @@ def env_csv(env_key: str, env_default = None) -> List[str]: [] :param str env_key: Environment var to attempt to load - :param any env_default: Fallback value if the env var is empty / not set + :param any env_default: Fallback value if the env var is empty / not set (Default: None) + :param str csvsplit: A character (or several) used to terminate each value in the list. Default: comma ``,`` :return List[str] parsed_data: A list of str values parsed from the env var """ d = env(env_key) - return env_default if empty(d) else parse_csv(d) + return env_default if empty(d) else parse_csv(d, csvsplit=csvsplit) + +def env_keyval(env_key: str, env_default = None, valsplit=':', csvsplit=',') -> List[Tuple[str, str]]: + """ + Parses an environment variable containing ``key:val,key:val`` into a list of tuples [(key,val), (key,val)] + + See :py:meth:`parse_keyval` + + :param str env_key: Environment var to attempt to load + :param any env_default: Fallback value if the env var is empty / not set (Default: None) + :param str valsplit: A character (or several) used to split the key from the value (default: colon ``:``) + :param str csvsplit: A character (or several) used to terminate each keyval pair (default: comma ``,``) + """ + d = env(env_key) + return env_default if empty(d) else parse_keyval(d, valsplit=valsplit, csvsplit=csvsplit) + +def env_bool(env_key: str, env_default = None) -> Union[bool, None]: + """ + Obtains an environment variable ``env_key``, if it's empty or not set, ``env_default`` will be returned. + Otherwise, it will be converted into a boolean using :py:func:`.is_true` + + Example: + + >>> os.environ['HELLO_WORLD'] = '1' + >>> env_bool('HELLO_WORLD') + True + >>> env_bool('HELLO_NOEXIST') + None + >>> env_bool('HELLO_NOEXIST', 'error') + 'error' + + :param str env_key: Environment var to attempt to load + :param any env_default: Fallback value if the env var is empty / not set (Default: None) + """ + return env_default if empty(env(env_key)) else is_true(env(env_key)) class ErrHelpParser(argparse.ArgumentParser): diff --git a/setup.py b/setup.py index 98c32ad..47a03c6 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup( name='privex_helpers', - version='1.1.2', + version='1.1.4', description='A variety of helper functions and classes, useful for many different projects', long_description=long_description, diff --git a/tests.py b/tests.py index 17d5048..e260470 100755 --- a/tests.py +++ b/tests.py @@ -46,8 +46,29 @@ OK +You can also use the ``pytest`` tool (used by default for our Travis CI):: -Copyright:: + user@host: ~/privex-helpers $ pip3 install pytest + # You can add `-v` for more detailed output, just like when running tests.py directly. + user@host: ~/privex-helpers $ pytest tests.py + + ===================================== test session starts ===================================== + platform darwin -- Python 3.7.0, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 + rootdir: /home/user/privex-helpers + collected 33 items + + tests.py ................................. [100%] + + ====================================== warnings summary ======================================= + /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/jinja2/utils.py:485 + /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/jinja2/utils.py:485: + DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' + is deprecated, and in 3.8 it will stop working + from collections import MutableMapping + ============================ 33 passed, 2 warnings in 0.17 seconds ============================ + + +**Copyright**:: Copyright 2019 Privex Inc. ( https://www.privex.io ) License: X11 / MIT Github: https://github.com/Privex/python-helpers @@ -56,12 +77,41 @@ """ import unittest import logging +import os from collections import namedtuple from privex import helpers from privex.loghelper import LogHelper from privex.helpers import ip_to_rdns, BoundaryException -class TestParseHelpers(unittest.TestCase): +class EmptyIter(object): + """A mock iterable object with zero length for testing empty()""" + def __len__(self): + return 0 + + +class PrivexBaseCase(unittest.TestCase): + """ + Base test-case for module test cases to inherit. + + Contains useful class attributes such as ``falsey`` and ``empty_vals`` that are used + across different unit tests. + """ + + falsey = ['false', 'FALSE', False, 0, '0', 'no'] + """Normal False-y values, as various types""" + + falsey_empty = falsey + [None, '', 'null'] + """False-y values, plus 'empty' values like '' and None""" + + truthy = [True, 'TRUE', 'true', 'yes', 'y', '1', 1] + """Truthful values, as various types""" + + empty_vals = [None, ''] + empty_lst = empty_vals + [[], (), set(), {}, EmptyIter()] + empty_zero = empty_vals + [0, '0'] + + +class TestParseHelpers(PrivexBaseCase): """Test the parsing functions parse_csv and parse_keyval""" def test_csv_spaced(self): @@ -93,23 +143,51 @@ def test_kval_single(self): helpers.parse_keyval('John:Doe'), [('John', 'Doe')] ) + + def test_kval_custom_clean(self): + """ + Test that a clean key:val csv with custom split characters is parsed correctly + (pipe for kv, semi-colon for pair separation) + """ + self.assertListEqual( + helpers.parse_keyval('John|Doe;Jane|Smith', valsplit='|', csvsplit=';'), + [('John', 'Doe'), ('Jane', 'Smith')] + ) + + def test_kval_custom_spaced(self): + """Test key:val csv parsing with excess outer/value whitespace, and custom split characters.""" + self.assertListEqual( + helpers.parse_keyval(' John | Doe ; Jane |Smith ', valsplit='|', csvsplit=';'), + [('John', 'Doe'), ('Jane', 'Smith')] + ) -class EmptyIter(object): - """A mock iterable object with zero length for testing empty()""" - def __len__(self): - return 0 + def test_env_nonexist_bool(self): + """Test env_bool returns default with non-existant env var""" + k = 'EXAMPLE_NOEXIST' + if k in os.environ: del os.environ[k] # Ensure the env var we're testing definitely does not exist. + self.assertIsNone(helpers.env_bool(k)) + self.assertEqual(helpers.env_bool(k, 'error'), 'error') + + def test_env_bool_true(self): + """Test env_bool returns True boolean with valid env var""" + k = 'EXAMPLE_EXIST' + for v in self.truthy: + os.environ[k] = str(v) + self.assertTrue(helpers.env_bool(k, 'fail'), msg=f'env_bool({v}) === True') + + def test_env_bool_false(self): + """Test env_bool returns False boolean with valid env var""" + k = 'EXAMPLE_EXIST' + for v in self.falsey: + os.environ[k] = str(v) + self.assertFalse(helpers.env_bool(k, 'fail'), msg=f'env_bool({v}) === False') -class TestBoolHelpers(unittest.TestCase): - """Test the boolean check functions is_true, is_false, as well as empty()""" - falsey = ['false', 'FALSE', None, False, '', 0, '0', 'no', 'null'] - truthy = [True, 'TRUE', 'true', 'yes', 'y', '1', 1] - empty_vals = [None, ''] - empty_lst = empty_vals + [[], (), set(), {}, EmptyIter()] - empty_zero = empty_vals + [0, '0'] +class TestBoolHelpers(PrivexBaseCase): + """Test the boolean check functions is_true, is_false, as well as empty()""" def test_isfalse_falsey(self): - for f in self.falsey: + for f in self.falsey_empty: self.assertTrue(helpers.is_false(f), msg=f"is_false({repr(f)}") def test_isfalse_truthy(self): @@ -121,7 +199,7 @@ def test_istrue_truthy(self): self.assertTrue(helpers.is_true(f), msg=f"is_true({repr(f)}") def test_istrue_falsey(self): - for f in self.falsey: + for f in self.falsey_empty: self.assertFalse(helpers.is_true(f), msg=f"!is_true({repr(f)}") def test_empty_vals(self): @@ -163,7 +241,7 @@ def test_notempty(self): VALID_V6_1_32BOUND = 'd.a.e.d.1.0.0.2.ip6.arpa' -class TestIPReverseDNS(unittest.TestCase): +class TestIPReverseDNS(PrivexBaseCase): """ Unit testing for the reverse DNS functions in :py:mod:`privex.helpers.net`