diff --git a/.github/workflows/run_tox.yml b/.github/workflows/run_tox.yml index 4fff190..ad69a07 100644 --- a/.github/workflows/run_tox.yml +++ b/.github/workflows/run_tox.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v1 diff --git a/doc/conf.py b/doc/conf.py index c1abaa8..f6f8d50 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -37,7 +37,7 @@ ] exec_code_working_dir = '../src' -exec_code_folders = ['../src'] +exec_code_source_folders = ['../src', '../src', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/doc/configuration.rst b/doc/configuration.rst index d3db30d..c0eab16 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -37,7 +37,7 @@ The following configuration parameters are available: - ``Path`` or ``str`` - The working directory where the code will be executed. - * - ``exec_code_folders`` + * - ``exec_code_source_folders`` - | ``List`` of | ``Path`` or ``str`` - | Additional folders that will be added to PYTHONPATH. @@ -48,10 +48,10 @@ The following configuration parameters are available: - | The directory that is used to create the path to the | example files. Defaults to the parent folder of the ``conf.py``. - * - ``exec_code_stdout_encoding`` - - ``str`` - - | Encoding used to decode stdout. - | The default depends on the operating system but should be ``utf-8``. + * - ``exec_code_set_utf8_encoding`` + - ``True`` or ``False`` + - | True enforces utf-8 encoding (can fix encoding errors). + | Default is ``False`` except on Windows where it is ``True``. If it's a relative path it will be resolved relative to the parent folder of the ``conf.py`` @@ -61,7 +61,7 @@ Example: .. code-block:: python exec_code_working_dir = '..' - exec_code_folders = ['../my_src'] + exec_code_source_folders = ['../my_src'] exec_code_example_dir = '.' If you are unsure what the values are you can run Sphinx build in verbose mode with ``-v -v``. @@ -72,6 +72,6 @@ Log output for Example: .. code-block:: text [exec-code] Working dir: C:\Python\sphinx-exec-code - [exec-code] Folders: C:\Python\sphinx-exec-code\my_src + [exec-code] Source folders: C:\Python\sphinx-exec-code\my_src [exec-code] Example dir: C:\Python\sphinx-exec-code\doc - [exec-code] Stdout encoding: utf-8 + [exec-code] Set utf8 encoding: True diff --git a/readme.md b/readme.md index d9fec87..ddd5744 100644 --- a/readme.md +++ b/readme.md @@ -36,9 +36,15 @@ This code will be executed ``` # Changelog +#### 0.8 (18.07.2022) +- Renamed ``exec_code_folders`` to ``exec_code_source_folders`` +- Changed type of parameter to specify stdout to a flag +- Changed default for config parameter that sets encoding +- Dropped support for Python 3.7 + #### 0.7 (15.07.2022) - Added config parameter to specify stdout encoding -- Only empty lines of the output get trimmed +- Only empty lines of the output get trimmed #### 0.6 (04.04.2022) - Fixed an issue where the line numbers for error messages were not correct diff --git a/setup.py b/setup.py index 6c03087..d233fc1 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,6 @@ def load_version() -> str: "License :: OSI Approved :: Apache Software License", "Natural Language :: English", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/src/sphinx_exec_code/__version__.py b/src/sphinx_exec_code/__version__.py index e220fa9..376df7c 100644 --- a/src/sphinx_exec_code/__version__.py +++ b/src/sphinx_exec_code/__version__.py @@ -1 +1 @@ -__version__ = '0.7' +__version__ = '0.8' diff --git a/src/sphinx_exec_code/code_exec.py b/src/sphinx_exec_code/code_exec.py index f21d0d3..7f35de9 100644 --- a/src/sphinx_exec_code/code_exec.py +++ b/src/sphinx_exec_code/code_exec.py @@ -3,40 +3,32 @@ import sys from itertools import dropwhile from pathlib import Path -from typing import Iterable, Optional - -from sphinx.errors import ConfigError from sphinx_exec_code.code_exec_error import CodeException - -WORKING_DIR: Optional[str] = None -ADDITIONAL_FOLDERS: Optional[Iterable[str]] = None -STDOUT_ENCODING: str = sys.stdout.encoding - - -def setup_code_env(cwd: Path, folders: Iterable[Path], encoding: str): - global WORKING_DIR, ADDITIONAL_FOLDERS, STDOUT_ENCODING - WORKING_DIR = str(cwd) - ADDITIONAL_FOLDERS = tuple(map(str, folders)) - STDOUT_ENCODING = encoding +from sphinx_exec_code.configuration import PYTHONPATH_FOLDERS, SET_UTF8_ENCODING, WORKING_DIR def execute_code(code: str, file: Path, first_loc: int) -> str: - if WORKING_DIR is None or ADDITIONAL_FOLDERS is None: - raise ConfigError('Working dir or additional folders are not set!') + cwd: Path = WORKING_DIR.value + encoding = 'utf-8' if SET_UTF8_ENCODING.value else None + python_folders = PYTHONPATH_FOLDERS.value env = os.environ.copy() - try: - env['PYTHONPATH'] = os.pathsep.join(ADDITIONAL_FOLDERS) + os.pathsep + env['PYTHONPATH'] - except KeyError: - env['PYTHONPATH'] = os.pathsep.join(ADDITIONAL_FOLDERS) - run = subprocess.run([sys.executable, '-c', code], capture_output=True, cwd=WORKING_DIR, env=env) + if python_folders: + try: + env['PYTHONPATH'] = os.pathsep.join(python_folders) + os.pathsep + env['PYTHONPATH'] + except KeyError: + env['PYTHONPATH'] = os.pathsep.join(python_folders) + + run = subprocess.run([sys.executable, '-c', code.encode('utf-8')], capture_output=True, text=True, + encoding=encoding, cwd=cwd, env=env) + if run.returncode != 0: - raise CodeException(code, file, first_loc, run.returncode, run.stderr.decode()) from None + raise CodeException(code, file, first_loc, run.returncode, run.stderr) from None # decode output and drop tailing spaces - ret_str = (run.stdout.decode(encoding=STDOUT_ENCODING) + run.stderr.decode(encoding=STDOUT_ENCODING)).rstrip() + ret_str = (run.stdout if run.stdout is not None else '' + run.stderr if run.stderr is not None else '').rstrip() # drop leading empty lines ret_lines = list(dropwhile(lambda x: not x.strip(), ret_str.splitlines())) diff --git a/src/sphinx_exec_code/code_format.py b/src/sphinx_exec_code/code_format.py index 9175e0a..dc46d7f 100644 --- a/src/sphinx_exec_code/code_format.py +++ b/src/sphinx_exec_code/code_format.py @@ -1,4 +1,4 @@ -from typing import Iterable, Tuple +from typing import Iterable, Tuple, List class VisibilityMarkerError(Exception): @@ -16,7 +16,7 @@ def __init__(self, marker: str): self.do_add = True self.skip_empty = False - self.lines = [] + self.lines: List[str] = [] def is_marker(self, line: str) -> bool: if line == self.start: diff --git a/src/sphinx_exec_code/configuration/__init__.py b/src/sphinx_exec_code/configuration/__init__.py new file mode 100644 index 0000000..a69964e --- /dev/null +++ b/src/sphinx_exec_code/configuration/__init__.py @@ -0,0 +1 @@ +from .values import EXAMPLE_DIR, PYTHONPATH_FOLDERS, SET_UTF8_ENCODING, WORKING_DIR diff --git a/src/sphinx_exec_code/configuration/base.py b/src/sphinx_exec_code/configuration/base.py new file mode 100644 index 0000000..d7672fd --- /dev/null +++ b/src/sphinx_exec_code/configuration/base.py @@ -0,0 +1,44 @@ +from typing import Any, Final, Generic, Optional, Tuple, Type, TypeVar, Union + +from sphinx.application import Sphinx as SphinxApp +from sphinx.errors import ConfigError + +from sphinx_exec_code.__const__ import log + +TYPE_VALUE = TypeVar('TYPE_VALUE') + + +class SphinxConfigValue(Generic[TYPE_VALUE]): + SPHINX_TYPE: Union[Tuple[Type[Any], ...], Type[Any]] + + def __init__(self, sphinx_name: str, initial_value: Optional[TYPE_VALUE] = None): + self.sphinx_name: Final = sphinx_name + self._value: Optional[TYPE_VALUE] = initial_value + + @property + def value(self) -> TYPE_VALUE: + if self._value is None: + raise ConfigError(f'{self.sphinx_name} is not set!') + return self._value + + def transform_value(self, app: SphinxApp, value): + return value + + def validate_value(self, value) -> TYPE_VALUE: + return value + + def from_app(self, app: SphinxApp) -> TYPE_VALUE: + # load value + value = self.transform_value(app, getattr(app.config, self.sphinx_name)) + + # log transformed value + assert self.sphinx_name.startswith('exec_code_') + name = self.sphinx_name[10:].replace('_', ' ').capitalize() + log.debug(f'[exec-code] {name:s}: {value}') + + # additional validation + self._value = self.validate_value(value) + return self._value + + def add_config_value(self, app: SphinxApp, sphinx_default): + app.add_config_value(self.sphinx_name, sphinx_default, 'env', self.SPHINX_TYPE) diff --git a/src/sphinx_exec_code/configuration/flag_config.py b/src/sphinx_exec_code/configuration/flag_config.py new file mode 100644 index 0000000..fd075b7 --- /dev/null +++ b/src/sphinx_exec_code/configuration/flag_config.py @@ -0,0 +1,10 @@ +from sphinx.application import Sphinx as SphinxApp + +from sphinx_exec_code.configuration.base import SphinxConfigValue + + +class SphinxConfigFlag(SphinxConfigValue[bool]): + SPHINX_TYPE = bool + + def transform_value(self, app: SphinxApp, value) -> bool: + return bool(value) diff --git a/src/sphinx_exec_code/configuration/path_config.py b/src/sphinx_exec_code/configuration/path_config.py new file mode 100644 index 0000000..05737e3 --- /dev/null +++ b/src/sphinx_exec_code/configuration/path_config.py @@ -0,0 +1,68 @@ +from pathlib import Path +from typing import Tuple + +from sphinx.application import Sphinx as SphinxApp + +from sphinx_exec_code.__const__ import log +from sphinx_exec_code.configuration.base import SphinxConfigValue, TYPE_VALUE + + +class SphinxConfigPath(SphinxConfigValue[TYPE_VALUE]): + SPHINX_TYPE = (str, Path) + + def make_path(self, app: SphinxApp, value) -> Path: + try: + path = Path(value) + except Exception: + raise ValueError(f'Could not create Path from "{value}" (type {type(value).__name__}) ' + f'(configured by {self.sphinx_name:s})') from None + + if not path.is_absolute(): + path = (Path(app.confdir) / path).resolve() + return path + + def check_folder_exists(self, folder: Path) -> Path: + if not folder.is_dir(): + raise FileNotFoundError(f'Directory "{folder}" not found! (configured by {self.sphinx_name:s})') + return folder + + +class SphinxConfigFolder(SphinxConfigPath[Path]): + def transform_value(self, app: SphinxApp, value) -> Path: + return self.make_path(app, value) + + def validate_value(self, value: Path) -> Path: + return self.check_folder_exists(value) + + +class SphinxConfigMultipleFolderStr(SphinxConfigPath[Tuple[str, ...]]): + SPHINX_TYPE = () + + def transform_value(self, app: SphinxApp, value) -> Tuple[Path, ...]: + return tuple(self.make_path(app, p) for p in value) + + def validate_value(self, value: Tuple[Path, ...]) -> Tuple[str, ...]: + # check that folders exist + for f in value: + self.check_folder_exists(f) + + # Search for a python package and print a warning if we find none + # since this is the only reason to specify additional folders + for f in value: + package_found = False + for _f in f.iterdir(): + if not _f.is_dir(): + continue + + # log warning if we don't find a python package + for file in _f.glob('__init__.py'): + if file.name == '__init__.py': + package_found = True + break + if package_found: + break + + if not package_found: + log.warning(f'[exec-code] No Python packages found in {f}') + + return tuple(map(str, value)) diff --git a/src/sphinx_exec_code/configuration/values.py b/src/sphinx_exec_code/configuration/values.py new file mode 100644 index 0000000..2ebddad --- /dev/null +++ b/src/sphinx_exec_code/configuration/values.py @@ -0,0 +1,9 @@ +from .flag_config import SphinxConfigFlag +from .path_config import SphinxConfigFolder, SphinxConfigMultipleFolderStr + +EXAMPLE_DIR = SphinxConfigFolder('exec_code_example_dir') + +# Options for code execution +WORKING_DIR = SphinxConfigFolder('exec_code_working_dir') +PYTHONPATH_FOLDERS = SphinxConfigMultipleFolderStr('exec_code_source_folders') +SET_UTF8_ENCODING = SphinxConfigFlag('exec_code_set_utf8_encoding') diff --git a/src/sphinx_exec_code/sphinx_api.py b/src/sphinx_exec_code/sphinx_api.py index 643fa65..f66370b 100644 --- a/src/sphinx_exec_code/sphinx_api.py +++ b/src/sphinx_exec_code/sphinx_api.py @@ -1,66 +1,21 @@ -import sys +import os from pathlib import Path +from sphinx.application import Sphinx as SphinxApp + from sphinx_exec_code import __version__ from sphinx_exec_code.__const__ import log -from sphinx_exec_code.code_exec import setup_code_env -from sphinx_exec_code.sphinx_exec import ExecCode, setup_example_dir - -CONF_NAME_CWD = 'exec_code_working_dir' -CONF_NAME_DIRS = 'exec_code_folders' -CONF_NAME_SAMPLE_DIR = 'exec_code_example_dir' -CONF_NAME_STDOUT_ENCODING = 'exec_code_stdout_encoding' - - -def mk_path(app, obj) -> Path: - confdir = Path(app.confdir) - path = Path(obj) - if not path.is_absolute(): - path = (confdir / path).resolve() - return path - +from sphinx_exec_code.sphinx_exec import ExecCode -def builder_ready(app): - cwd = mk_path(app, getattr(app.config, CONF_NAME_CWD)) - folders = tuple(mk_path(app, _p) for _p in getattr(app.config, CONF_NAME_DIRS)) - example_dir = mk_path(app, getattr(app.config, CONF_NAME_SAMPLE_DIR)) - stdout_encoding = getattr(app.config, CONF_NAME_STDOUT_ENCODING) +from .configuration import EXAMPLE_DIR, PYTHONPATH_FOLDERS, SET_UTF8_ENCODING, WORKING_DIR - log.debug(f'[exec-code] Working dir: {cwd}') - log.debug(f'[exec-code] Folders: {", ".join(map(str, folders))}') - log.debug(f'[exec-code] Example dir: {example_dir}') - log.debug(f'[exec-code] Stdout encoding: {stdout_encoding}') - # Ensure dirs are valid - if not cwd.is_dir(): - raise FileNotFoundError(f'Working directory "{cwd}" not found! (configured by {CONF_NAME_CWD})') - if not example_dir.is_dir(): - raise FileNotFoundError(f'Example directory "{example_dir}" not found! (configured by {CONF_NAME_SAMPLE_DIR})') - for _f in folders: - if not _f.is_dir(): - raise FileNotFoundError(f'Additional directory "{_f}" not found! (configured by {CONF_NAME_DIRS})') - - # Search for a python package and print a warning if we find none - # since this is the only reason to specify additional folders - for _f in folders: - package_found = False - for __f in _f.iterdir(): - if not __f.is_dir(): - continue - - # log warning if we don't find a python package - for file in __f.iterdir(): - if file.name == '__init__.py': - package_found = True - break - if package_found: - break - - if not package_found: - log.warning(f'[exec-code] No Python packages found in {_f}') - - setup_example_dir(example_dir) - setup_code_env(cwd, folders, stdout_encoding) +def builder_ready(app: SphinxApp): + # load configuration + EXAMPLE_DIR.from_app(app) + WORKING_DIR.from_app(app) + PYTHONPATH_FOLDERS.from_app(app) + SET_UTF8_ENCODING.from_app(app) return None @@ -69,18 +24,16 @@ def setup(app): confdir = Path(app.confdir) - cwd = str(confdir.parent) - code_folders = [] src_dir = confdir.with_name('src') if src_dir.is_dir(): - code_folders.append(str(src_dir)) + code_folders.append(src_dir) - # config options - app.add_config_value(CONF_NAME_CWD, cwd, 'env', (Path, str)) - app.add_config_value(CONF_NAME_DIRS, code_folders, 'env', (Path, str)) - app.add_config_value(CONF_NAME_SAMPLE_DIR, confdir, 'env', (Path, str)) - app.add_config_value(CONF_NAME_STDOUT_ENCODING, sys.stdout.encoding, 'env', str) + # Configuration options + EXAMPLE_DIR.add_config_value(app, confdir) + WORKING_DIR.add_config_value(app, confdir.parent) + PYTHONPATH_FOLDERS.add_config_value(app, code_folders) + SET_UTF8_ENCODING.add_config_value(app, True if os.name == 'nt' else False) # Somehow this workaround is required app.connect('builder-inited', builder_ready) app.add_directive('exec_code', ExecCode) @@ -91,6 +44,6 @@ def setup(app): 'version': __version__, # https://github.com/spacemanspiff2007/sphinx-exec-code/issues/2 - # This extension does not store any states so it should be safe for parallel reading + # This extension does not store any states making it safe for parallel reading 'parallel_read_safe': True } diff --git a/src/sphinx_exec_code/sphinx_exec.py b/src/sphinx_exec_code/sphinx_exec.py index abdc1b7..e8b7e83 100644 --- a/src/sphinx_exec_code/sphinx_exec.py +++ b/src/sphinx_exec_code/sphinx_exec.py @@ -1,6 +1,6 @@ import traceback from pathlib import Path -from typing import List, Optional +from typing import List from docutils import nodes from sphinx.errors import ExtensionError @@ -9,15 +9,9 @@ from sphinx_exec_code.__const__ import log from sphinx_exec_code.code_exec import CodeException, execute_code from sphinx_exec_code.code_format import get_show_exec_code, VisibilityMarkerError +from sphinx_exec_code.configuration import EXAMPLE_DIR from sphinx_exec_code.sphinx_spec import build_spec, SpecCode, SpecOutput, SphinxSpecBase -EXAMPLE_DIR: Optional[Path] = None - - -def setup_example_dir(example_dir: Path): - global EXAMPLE_DIR - EXAMPLE_DIR = example_dir - def create_literal_block(objs: list, code: str, spec: SphinxSpecBase): if spec.hide or not code: @@ -91,7 +85,7 @@ def _run(self) -> list: # Read from example files if code_spec.filename: - filename = (EXAMPLE_DIR / code_spec.filename).resolve() + filename = (EXAMPLE_DIR.value / code_spec.filename).resolve() content = filename.read_text(encoding='utf-8').splitlines() file, line = filename, 1 diff --git a/src/sphinx_exec_code/sphinx_spec.py b/src/sphinx_exec_code/sphinx_spec.py index ac18101..0bbbd9a 100644 --- a/src/sphinx_exec_code/sphinx_spec.py +++ b/src/sphinx_exec_code/sphinx_spec.py @@ -5,7 +5,7 @@ class SphinxSpecBase: aliases: Dict[str, str] - defaults = Dict[str, str] + defaults: Dict[str, str] def __init__(self, hide: bool, linenos: bool, caption: str, language: str): # flags diff --git a/tests/test_code_exec.py b/tests/test_code_exec.py index cd5b543..d387b68 100644 --- a/tests/test_code_exec.py +++ b/tests/test_code_exec.py @@ -1,27 +1,34 @@ +import os from pathlib import Path import pytest -import sphinx_exec_code.code_exec from sphinx_exec_code.code_exec import CodeException, execute_code +from sphinx_exec_code.configuration import PYTHONPATH_FOLDERS, SET_UTF8_ENCODING, WORKING_DIR @pytest.fixture def setup_env(monkeypatch): f = Path(__file__).parent - monkeypatch.setattr(sphinx_exec_code.code_exec, 'WORKING_DIR', str(f)) - monkeypatch.setattr(sphinx_exec_code.code_exec, 'ADDITIONAL_FOLDERS', [str(f)]) + monkeypatch.setattr(WORKING_DIR, '_value', f) + monkeypatch.setattr(PYTHONPATH_FOLDERS, '_value', [str(f)]) yield -def test_print(setup_env): +@pytest.mark.parametrize('utf8', [True, False]) +def test_print(setup_env, monkeypatch, utf8): + monkeypatch.setattr(SET_UTF8_ENCODING, '_value', utf8) + code = "print('Line1')\nprint('Line2')" output = execute_code(code, 'my_file', 1) assert output == 'Line1\nLine2' -def test_print_table(setup_env): +@pytest.mark.parametrize('utf8', [True, False]) +def test_print_table(setup_env, monkeypatch, utf8): + monkeypatch.setattr(SET_UTF8_ENCODING, '_value', utf8) + code = "\n \n \n\n" \ "print(' | A | B |')\n" \ "print(' Col1 | 1 | 2 |')" @@ -29,7 +36,10 @@ def test_print_table(setup_env): assert output == ' | A | B |\n Col1 | 1 | 2 |' -def test_err(setup_env): +@pytest.mark.parametrize('utf8', [True, False]) +def test_err(setup_env, monkeypatch, utf8): + monkeypatch.setattr(SET_UTF8_ENCODING, '_value', utf8) + code = "print('Line1')\nprint('Line2')\n1/0" with pytest.raises(CodeException) as e: @@ -47,7 +57,26 @@ def test_err(setup_env): ] -def test_unicode(setup_env): +IS_WIN = os.name == 'nt' + + +@pytest.mark.skipif(not IS_WIN, reason='Windows only') +def test_unicode_fails(setup_env, monkeypatch): code = "print('●')" - output = execute_code(code, 'my_file', 1) - assert output == '●' + + monkeypatch.setattr(SET_UTF8_ENCODING, '_value', False) + assert execute_code(code, 'my_file', 1) != '●' + + monkeypatch.setattr(SET_UTF8_ENCODING, '_value', True) + assert execute_code(code, 'my_file', 1) == '●' + + +@pytest.mark.skipif(IS_WIN, reason='Fails on Windows') +def test_unicode_no_utf8(setup_env, monkeypatch): + code = "print('●')" + + monkeypatch.setattr(SET_UTF8_ENCODING, '_value', False) + assert execute_code(code, 'my_file', 1) == '●' + + monkeypatch.setattr(SET_UTF8_ENCODING, '_value', True) + assert execute_code(code, 'my_file', 1) == '●' diff --git a/tests/test_sphinx_config.py b/tests/test_sphinx_config.py new file mode 100644 index 0000000..424918e --- /dev/null +++ b/tests/test_sphinx_config.py @@ -0,0 +1,17 @@ +from pathlib import Path + +import pytest + +from sphinx_exec_code.configuration.values import SphinxConfigFolder + + +def test_path_errors(): + a = SphinxConfigFolder('config_key_name') + + with pytest.raises(FileNotFoundError) as e: + a.check_folder_exists(Path('does_not_exist')) + assert str(e.value) == 'Directory "does_not_exist" not found! (configured by config_key_name)' + + with pytest.raises(ValueError) as e: + a.make_path(None, 1) + assert str(e.value) == 'Could not create Path from "1" (type int) (configured by config_key_name)'