From 438381a447e08d5e8a54675dcd2ec926fb00fc7b Mon Sep 17 00:00:00 2001 From: Olivier Ramonat Date: Wed, 20 Mar 2024 15:49:30 +0100 Subject: [PATCH 1/3] Create a pytest plugin to reuse fixtures in other projects Remove deprecated requirements traceability which is not maintained. Create e3.pytest to reuse part of the code in conftest.py in other projects. Document the new plugin. --- .github/workflows/ci.yaml | 2 +- .gitignore | 1 + NEWS.md | 2 +- docs/generate-req-coverage.py | 50 ------ docs/source/index.rst | 2 +- docs/source/pytest.rst | 85 ++++++++++ docs/source/requirements.rst | 5 - docs/source/requirements.yaml | 48 ------ pyproject.toml | 20 +++ src/e3/pytest.py | 223 +++++++++++++++++++++++++ tests/conftest.py | 31 ---- tests/coverage/base.rc | 20 --- tests/coverage/darwin.rc | 4 - tests/coverage/linux.rc | 4 - tests/coverage/omit-files-darwin | 2 + tests/coverage/omit-files-linux | 2 + tests/coverage/omit-files-windows | 1 + tests/coverage/windows.rc | 3 - tests/fix-coverage-paths.py | 35 ---- tests/gen-cov-config.py | 80 --------- tests/tests_e3/anod/test_buildspace.py | 5 +- tests/tests_e3/conftest.py | 166 +----------------- tests/tests_e3/vcs/git/main_test.py | 6 +- tests/tests_e3/vcs/svn/main_test.py | 3 +- tox.ini | 21 +-- 25 files changed, 348 insertions(+), 473 deletions(-) delete mode 100644 docs/generate-req-coverage.py create mode 100644 docs/source/pytest.rst delete mode 100644 docs/source/requirements.rst delete mode 100644 docs/source/requirements.yaml create mode 100644 src/e3/pytest.py delete mode 100644 tests/conftest.py delete mode 100644 tests/coverage/base.rc delete mode 100644 tests/coverage/darwin.rc delete mode 100644 tests/coverage/linux.rc create mode 100644 tests/coverage/omit-files-darwin create mode 100644 tests/coverage/omit-files-linux create mode 100644 tests/coverage/omit-files-windows delete mode 100644 tests/coverage/windows.rc delete mode 100644 tests/fix-coverage-paths.py delete mode 100644 tests/gen-cov-config.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d210b317..93f425de 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,7 +31,7 @@ jobs: - name: Run Tox run: tox env: - TOXENV: py${{ matrix.python-version}}-ci-xdist-cov + TOXENV: py${{ matrix.python-version}}-xdist-cov security: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index a6f44c47..77a76c88 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__ /.cache /.coverage* /.coveragerc +/coverage.xml /.idea /dist/ /build/ diff --git a/NEWS.md b/NEWS.md index fa4f78a3..0568eaae 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Version 22.5.0 (2023-??-??) *NOT RELEASED YET* -* Nothing +* Add e3.pytest plugin to reuse fixtures in other projects # Version 22.4.0 (2023-01-18) diff --git a/docs/generate-req-coverage.py b/docs/generate-req-coverage.py deleted file mode 100644 index 3b3c36e6..00000000 --- a/docs/generate-req-coverage.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# type: ignore -"""Generate the requirement coverage for e3-core doc.""" - -import sys -import yaml - - -def lookup(item, d): - """Lookup an item in a dictionary.""" - return {key for key, value in d.items() if value == item} - - -def merge_docs(requirement: str, coverage: str) -> dict: - """Load requirement yaml and include the coverage info. - - :param requirement: yaml filename containing list of requirements - :param coverage: yaml filename formatting as 'testname: requirement' - """ - with open(requirement) as f: - reqs = yaml.safe_load(f) - - with open(coverage) as f: - reqs_cov = yaml.safe_load(f) - - for k in reqs: - tests = lookup(k, reqs_cov) - reqs[k]["tests"] = tests - return reqs - - -def generate_rst(reqs_result: dict, dest: str): - """Generate rst file from requirement coverage data. - - :param reqs_result: dictionary returned by merge_docs - :param dest: rst file to create - """ - with open(dest, "w") as f: - for k in reqs_result: - f.write("- %s\n" % k) - f.write(" %s\n" % reqs_result[k]["desc"].encode("utf-8")) - tests = reqs_result[k]["tests"] - if tests: - f.write(" **Covered by %s**\n" % ", ".join(reqs_result[k]["tests"])) - else: - f.write(" **Not yet covered**\n") - - -if __name__ == "__main__": - generate_rst(merge_docs(sys.argv[1], sys.argv[2]), sys.argv[3]) diff --git a/docs/source/index.rst b/docs/source/index.rst index 9767c921..8b753680 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,7 +14,7 @@ Welcome to e3-core's documentation! quickstart decision_graph plugins - requirements + pytest diff --git a/docs/source/pytest.rst b/docs/source/pytest.rst new file mode 100644 index 00000000..b6640efa --- /dev/null +++ b/docs/source/pytest.rst @@ -0,0 +1,85 @@ +Using the e3 pytest plugin +========================== + +Introduction +------------ + +``e3-core`` contains a ``pytest`` plugin that is discovered automatically when +installed. The plugin provides several features: it generates a results file +compatible with anod when the environment variable ``RESULTS_DIR`` is defined. +It provides a simple setup for running coverage (on top of the pytest-cov +plugin). And it provides a env_protect fixture that is automatically activated. + + +Activating e3-core pytest plugins +--------------------------------- + +To activate the e3-core pytest plugin, you need to install e3-core and pass +the option ``--e3`` to pytest. + +env_protect +^^^^^^^^^^^ + +When activated, the plugin will register the ``env_protect`` feature to ensure +that all tests are run in isolation. All changes to the environment done in +each test won't impact other tests. Also, each test is run in a separate temp +directory, you won't have to cleanup the files that the tests create. + +``env_protect`` also sets some environment variables such as: + +* ``TZ=UTC`` to ensure a consistent timezone handling +* ``E3_ENABLE_FEATURE=""`` to discard any specific features supported by e3 +* ``E3_CONFIG=/dev/null`` to avoid having a specific e3 config read by the tests + +And the e3 DEBUG log level is activated for each tests. + +Coverage +^^^^^^^^ + +When running pytest with ``--e3`` and ``--cov`` options, pytest will +automatically generate an exclude list for lines matching the following +patterns: + +* ``all: no cover`` +* ``if TYPE_CHECKING:`` +* ``@abstractmethod`` +* ``# os-specific`` +* ``defensive code`` +* ``assert_never(),`` + +And ``-only`` with ```` different from the local OS, so if you're +running a test on Linux, ``windows-only`` and ``darwin-only`` will be discared. + +The opposite ``: no cover`` is also supported. + +Specific test for the windows platform are also detected: + +* ``if sys.platform == win32`` +* ``if sys.platform != win32`` +* ``unix-only`` + +You can also skip complete files by creating an ``omit file`` in +``tests/coverage/omit-file-``. The file should contain a filename per line. + +Finally, the option ``--e3-cov-rewrite `` changes the paths +reported by coverage. If you run ``--e3-cov-rewrite +.tox/py311/cov-xdist/lib-site-packages src`` instead of seeing reports of files in +``.tox/py311-cov-xdist/lib/site-packages/e3/`` the report will show files +in the repository ``src/e3/``. + +``require_tool`` fixture +------------------------ + +``e3.pytest`` provides a function ``require_tool`` that generates a fixture +allowing to skip tests if a tool is missing. For instance, to create a fixture +that will skip tests if ``git`` is not installed run: + +.. code-block:: python + + from e3.pytest import require_tool + + git = require_tool("git") + + # Use it in a test that will run only if git is installed + def test_git_fixture(git): + ... diff --git a/docs/source/requirements.rst b/docs/source/requirements.rst deleted file mode 100644 index 10412895..00000000 --- a/docs/source/requirements.rst +++ /dev/null @@ -1,5 +0,0 @@ -e3 software requirements -======================== - - -.. include:: requirement_coverage.rst diff --git a/docs/source/requirements.yaml b/docs/source/requirements.yaml deleted file mode 100644 index 827dd962..00000000 --- a/docs/source/requirements.yaml +++ /dev/null @@ -1,48 +0,0 @@ -REQ-EC1: - desc: Failing to write in the local filesystem causes an anod action to be requeued -REQ-EC2: - desc: All sources used in the context of an anod action are tracked in the build space -REQ-EC3: - desc: electrolyt tracks the origin and license of all sources -REQ-EC4: - desc: two identical anod commands always use the same build space -REQ-EC5: - desc: after a specification file update, an anod action using that specification file is never skipped -REQ-EC6: - desc: after a configuration file update, an anod action using that specification file is never skipped -REQ-EC7: - desc: For a given anod spec, an update of any of its dependencies, an anod action using that spec is never skipped. -REQ-EC8: - desc: For a given anod spec, an update of any of its resources, an anod action using that spec is never skipped. -REQ-EC9: - desc: For a given anod spec, running a second time an anod action which has failed does never result in a SKIP. -REQ-EC10: - desc: After an anod Spec API update, an anod action is never skipped. -REQ-EC11: - desc: After a build OS version update (Eg. update of build machine from RH5 to RH5.5), an anod action is never skipped. -REQ-EC12: - desc: For each repository associated with a source used in a given anod spec, when the patch applicable to that repository has been added/changed/removed, any action using that anod spec is never skipped. -REQ-EC13: - desc: For each repository associated with a source used in a given anod spec, when the repository has been modified, any action that rely on this repository is never skipped. -REQ-EC14: - desc: After a change in a BuildVar an anod action does never result in a SKIP. -REQ-EC15: - desc: An unhandled exception occurring inside an anod spec is caught by the anod driver, logged, and the status of the anod action is set to FAILED. -REQ-EC16: - desc: The list of anod actions from the expansion of an electrolyt plan does not contain any duplicate. -REQ-EC17: - desc: During the execution of an anod action, reading an invalid state or cache contents of the build space results in the reset of the current build space. -REQ-EC18: - desc: Read-Only Primitives/Actions do not modify the sandbox -REQ-EC19: - desc: Any non-Read-Only anod Action starts by deleting the tmp directory in the corresponding build space. -REQ-EC20: - desc: Calling anod with a qualifier which has not been declared inside the anod spec causes anod to fail. - note: At the implementation level, the expectation is that all qualifiers that an anod spec are declared and documented in the spec, so anod is able to determine whether each qualifier is valid or not. This allows the detection of typos in the qualifier. -REQ-EC21: - desc: Qualifier keys and values is a valid directory name on all supported platforms. They also do not contain ‘, space, nor ‘=’.. - note: To be defined precisely. Alphanumeric + _ and - ? -REQ-EC22: - desc: Applying a patch were one hunk was already applied should fail with a clear error message. -REQ-EC23: - desc: When a patch not apply, anod should report the conflict and fail. diff --git a/pyproject.toml b/pyproject.toml index 9e6670b3..c18f3e4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ test = [ ] check = [ "mypy==1.8.0", + "pytest", # for the pytest plugin "bandit", "pip-audit", "types-colorama", @@ -97,9 +98,28 @@ file-cache = "e3.store.cache.backends.filecache:FileCache" [project.entry-points."sandbox_scripts"] anod = "e3.anod.sandbox.scripts:anod" +[project.entry-points."pytest11"] +pytest = "e3.pytest" + [tool.setuptools.dynamic] version = {file = "VERSION"} +[tool.coverage.report] +fail_under = 90 + +[tool.coverage.run] +branch = true +omit = [ + "*mypy.py" +] + +[tool.coverage.html] +title = "e3 coverage report" + +[tool.pytest.ini_options] +addopts = "--failed-first --disable-socket --e3" + + [tool.mypy] # Ensure mypy works with namespace in which there is no toplevel # __init__.py. Explicit_package_bases means that that mypy_path diff --git a/src/e3/pytest.py b/src/e3/pytest.py new file mode 100644 index 00000000..3d801f4c --- /dev/null +++ b/src/e3/pytest.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from tempfile import mkdtemp, NamedTemporaryFile + +import logging +import os + +from e3.config import Config +from e3.env import Env +from e3.fs import rm +from e3.os.fs import cd, mv, which + +import e3.log + +from coverage.sqldata import CoverageData +from coverage.files import PathAliases +from coverage import Coverage + + +import pytest + +import typing + +if typing.TYPE_CHECKING: + from typing import Callable + + +test_errors = False + +# Detect that we're in CI mode, most providers set the $CI environment variable +IN_CI_MODE = "CI" in os.environ + +DEFAULT_EXCLUDE_LIST = ( + "all: no cover", + "if TYPE_CHECKING:", + "@abstractmethod", + "# os-specific", + "defensive code", + "assert_never()", +) + + +def require_tool(toolname: str) -> Callable: + """Require a specific tool to run the test. + + When in "CI" mode, a missing tool generates an error. In other + modes the test is just skipped. + + :param toolname: name of a tool, e.g. git + """ + + def wrapper(request: pytest.FixtureRequest) -> None: + if not which(toolname): + if IN_CI_MODE: + pytest.fail(f"{toolname} not available") + else: + pytest.skip(f"{toolname} not available") + + return pytest.fixture(wrapper) + + +def pytest_addoption( + parser: pytest.Parser, pluginmanager: pytest.PytestPluginManager +) -> None: + group = parser.getgroup("e3") + group.addoption("--e3", action="store_true", help="Use e3 fixtures and reporting") + group.addoption("--e3-cov-rewrite", nargs=2, help="Use e3 fixtures and reporting") + + +@pytest.fixture(autouse=True) +def env_protect(request: pytest.FixtureRequest) -> None: + """Protection against environment change. + + The fixture is enabled for all tests and does the following: + + * store/restore env between each tests + * create a temporary directory and do a cd to it before each + test. The directory is automatically removed when test ends + """ + if request.config.getoption("e3"): + Env().store() + tempd = mkdtemp() + cd(tempd) + Config.data = {} + + os.environ["TZ"] = "UTC" + os.environ["E3_ENABLE_FEATURE"] = "" + os.environ["E3_CONFIG"] = "/dev/null" + if "E3_HOSTNAME" in os.environ: + del os.environ["E3_HOSTNAME"] + + def restore_env() -> None: + Env().restore() + rm(tempd, True) + + e3.log.activate(level=logging.DEBUG, e3_debug=True) + request.addfinalizer(restore_env) + + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + """Manage the exit code depending on if errors were detected or not.""" + if not session.config.getoption("e3"): + return + global test_errors + if test_errors: + # Return with an exit code of `3` if we encountered errors (not failures). + # This is the exit code that corresponds to an "internal error" according to the + # pytest docs, which is the closest thing to having an actual Python error in + # test code. + session.exitstatus = 3 + + if session.config.getoption("cov_source"): + cov_file = str(session.config.rootpath / ".coverage") + if session.config.getoption("e3_cov_rewrite"): + origin_dir, new_dir = session.config.getoption("e3_cov_rewrite") + fix_coverage_paths(origin_dir=origin_dir, new_dir=new_dir, cov_db=cov_file) + + os_name = Env().build.os.name + + cov = Coverage(data_file=cov_file) + + # Exclude all lines matching the DEFAULT_EXCLUDE_LIST + for regex in DEFAULT_EXCLUDE_LIST: + cov.exclude(regex) + + # Exclude all -only line (unless that's for this OS) + for regex in ( + "%s-only" % o + for o in ("darwin", "linux", "solaris", "windows", "bsd", "aix") + if o != os_name + ): + cov.exclude(regex) + + # Handling of Windows + if os_name == "windows": + cov.exclude('if sys.platform != "win32":') + else: + cov.exclude('if sys.platform == "win32":') + cov.exclude("unix: no cover") + + # Exclude no cover for this OS + cov.exclude(f"{os_name}: no cover") + + cov.load() + + # Read configuration files in /tests/coverage to fix the list + # of files to omit for OS specific list + # We're relying on the default coverage configuration for the + # platform-agnostic list + omit_files: list[str] = cov.get_option("run:omit") or [] # type: ignore + coverage_conf_dir = session.config.rootpath / "tests" / "coverage" + conf_file = coverage_conf_dir / f"omit-files-{os_name}" + if conf_file.exists(): + with conf_file.open() as f: + for line in f: + omit_files.append(line.rstrip()) + + # cov.html_report(directory=str(session.config.rootpath ), omit=omit_files) + cov.html_report(omit=omit_files) + cov.report(omit=omit_files, precision=3) + cov.xml_report() + + +def fix_coverage_paths(origin_dir: str, new_dir: str, cov_db: str) -> None: + """Fix coverage paths. + + :param origin_dir: path to the package directory, e.g. + .tox/py311-cov-xdist/lib/python3.11/site-packages/e3 + :param new_dir: path to the dir that should be visible instead of origin_dir + e.g. src/ + :param cov_db: path to the .coverage data base + """ + paths = PathAliases() + paths.add(origin_dir, new_dir) + + old_cov_file = NamedTemporaryFile(dir=os.path.dirname(cov_db)) + old_cov_file.close() + try: + mv(cov_db, old_cov_file.name) + + old_coverage_data = CoverageData(old_cov_file.name) + old_coverage_data.read() + new_coverage_data = CoverageData(cov_db) + new_coverage_data.update(old_coverage_data, aliases=paths) + new_coverage_data.write() + finally: + os.unlink(old_cov_file.name) + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None]) -> None: # type: ignore + """Generate results file. + + When the variable results_dir is set to an existing directory, the testsuite + will generate results file in "anod" format. + """ + global test_errors + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + results_dir = os.environ.get("RESULTS_DIR") + + if not results_dir or not os.path.isdir(results_dir): + return + + # we only look at actual test calls, not setup/teardown + if rep.when == "call": + outcome = rep.outcome.upper() + test_name = rep.nodeid.replace("/", ".").replace("::", "--") + if rep.longreprtext: + with open(os.path.join(results_dir, f"{test_name}.diff"), "w") as f: + f.write(rep.longreprtext) + + with open(os.path.join(results_dir, "results"), "a") as f: + f.write(f"{test_name}:{outcome}\n") + else: + # If we detect a failure in an item that is not a "proper" test call, it's most + # likely an error. + # For example, this could be a failing assertion or a syntax error in a + # setup/teardown context. + if rep.outcome == "failed": + test_errors = True diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 9fe32e81..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,31 +0,0 @@ -# type: ignore -import logging -import os - -import e3.log - - -def init_testsuite_env(): - """Initialize testsuite environment.""" - # Activate full debug logs - e3.log.activate(level=logging.DEBUG, e3_debug=True) - - # Force UTC timezone - os.environ["TZ"] = "UTC" - os.environ["E3_ENABLE_FEATURE"] = "smtp_ssl" - os.environ["E3_CONFIG"] = "/dev/null" - # Ignore E3_HOSTNAME variable - if "E3_HOSTNAME" in os.environ: - del os.environ["E3_HOSTNAME"] - - -init_testsuite_env() - - -def pytest_addoption(parser): - parser.addoption( - "--ci", action="store_true", help="Tests are running on a CI server" - ) - parser.addoption( - "--requirement-coverage-report", help="Report requirement coverage" - ) diff --git a/tests/coverage/base.rc b/tests/coverage/base.rc deleted file mode 100644 index 55f0a01f..00000000 --- a/tests/coverage/base.rc +++ /dev/null @@ -1,20 +0,0 @@ -[run] -branch = False -# we can probably activate it once we have more coverage - -[report] -fail_under = 90 -omit = - *mypy.py -exclude_lines = - all: no cover - if TYPE_CHECKING: - @abstractmethod - # os-specific - defensive code - assert_never() - # + -only and : no cover - - -[html] -title = e3 coverage report diff --git a/tests/coverage/darwin.rc b/tests/coverage/darwin.rc deleted file mode 100644 index a728d1b7..00000000 --- a/tests/coverage/darwin.rc +++ /dev/null @@ -1,4 +0,0 @@ -[run] -omit = - */os/windows/* - */os/unix/constant.py diff --git a/tests/coverage/linux.rc b/tests/coverage/linux.rc deleted file mode 100644 index a728d1b7..00000000 --- a/tests/coverage/linux.rc +++ /dev/null @@ -1,4 +0,0 @@ -[run] -omit = - */os/windows/* - */os/unix/constant.py diff --git a/tests/coverage/omit-files-darwin b/tests/coverage/omit-files-darwin new file mode 100644 index 00000000..60353912 --- /dev/null +++ b/tests/coverage/omit-files-darwin @@ -0,0 +1,2 @@ +*/os/windows/* +*/os/unix/constant.py diff --git a/tests/coverage/omit-files-linux b/tests/coverage/omit-files-linux new file mode 100644 index 00000000..60353912 --- /dev/null +++ b/tests/coverage/omit-files-linux @@ -0,0 +1,2 @@ +*/os/windows/* +*/os/unix/constant.py diff --git a/tests/coverage/omit-files-windows b/tests/coverage/omit-files-windows new file mode 100644 index 00000000..096d7ec5 --- /dev/null +++ b/tests/coverage/omit-files-windows @@ -0,0 +1 @@ +*/os/unix/* diff --git a/tests/coverage/windows.rc b/tests/coverage/windows.rc deleted file mode 100644 index fc251567..00000000 --- a/tests/coverage/windows.rc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -omit = - */os/unix/* diff --git a/tests/fix-coverage-paths.py b/tests/fix-coverage-paths.py deleted file mode 100644 index e49e5b37..00000000 --- a/tests/fix-coverage-paths.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# strip .tox/*/lib/python*/site-packages paths from coverage data -# show only paths corresponding to the real source files -# From https://github.com/danilobellini/pytest-doctest-custom/ -# type: ignore - - -import os -import shutil -import sys - -from coverage.sqldata import CoverageData -from coverage.files import PathAliases -from tempfile import NamedTemporaryFile - - -def fix_paths(site_pkg_dir, cov_data_file): - site_pkg_dir = os.path.abspath(site_pkg_dir) - - paths = PathAliases() - paths.add(site_pkg_dir, "src") - - old_cov_file = NamedTemporaryFile() - old_cov_file.close() - shutil.move(cov_data_file, old_cov_file.name) - - old_coverage_data = CoverageData(old_cov_file.name) - old_coverage_data.read() - new_coverage_data = CoverageData(cov_data_file) - new_coverage_data.update(old_coverage_data, aliases=paths) - new_coverage_data.write() - - -if __name__ == "__main__": - fix_paths(sys.argv[1], sys.argv[2]) diff --git a/tests/gen-cov-config.py b/tests/gen-cov-config.py deleted file mode 100644 index e557a134..00000000 --- a/tests/gen-cov-config.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# type: ignore - - -from argparse import ArgumentParser -import os - -from e3.env import Env - -from configparser import ConfigParser - - -def main(coverage_rc, omit_list_filename=None): - os_name = Env().build.os.name - test_dir = os.path.abspath(os.path.dirname(__file__)) - - config = ConfigParser() - base_conf, target_conf = ( - os.path.join(test_dir, "coverage", "%s.rc" % name) for name in ("base", os_name) - ) - - with open(coverage_rc, "w") as dest: - config.read(base_conf) - config.read(target_conf) - - # exclude lines is built with: base.rc config - exclude_lines = config.get("report", "exclude_lines").splitlines() - - # add all -only patterns - exclude_lines += [ - "%s-only" % o - for o in ("darwin", "linux", "solaris", "windows", "bsd", "aix") - if o != os_name - ] - if os_name != "windows": - exclude_lines.append('if sys.platform == "win32":') - # exclude this specific os - exclude_lines.append("%s: no cover" % os_name) - - # special case for unix - if os_name != "windows": - exclude_lines.append("unix: no cover") - - config.set("report", "exclude_lines", "\n".join(exclude_lines)) - - # If the user gave a file with a list of "omit" entries, - # read it, and append its contents to the run.omit option. - - if omit_list_filename is not None: - if config.has_option("run", "omit"): - omit = config.get("run", "omit").splitlines() - else: - omit = [] - with open(omit_list_filename) as f: - for line in f: - omit.append(line) - config.set("run", "omit", "\n".join(omit)) - - config.write(dest) - - -if __name__ == "__main__": - parser = ArgumentParser(description="Generate a coverage config file") - parser.add_argument( - "coverage_rc_filename", help="The name of the coverage configuration file." - ) - parser.add_argument( - "--omit-from-file", - dest="omit_list_filename", - help=( - "The name of the file providing a list of files which should" - " be excluded in the coverage report, with each line being" - " a glob pattern matching the files that should be omitted." - " This omit list is in addition to the list of files to" - " be omitted by default." - ), - ) - - args = parser.parse_args() - main(args.coverage_rc_filename, omit_list_filename=args.omit_list_filename) diff --git a/tests/tests_e3/anod/test_buildspace.py b/tests/tests_e3/anod/test_buildspace.py index 946b646d..91edf930 100644 --- a/tests/tests_e3/anod/test_buildspace.py +++ b/tests/tests_e3/anod/test_buildspace.py @@ -32,10 +32,7 @@ def test_subdir(): def test_reset_tmp_dir(): - """Check that the tmp_dir is reset when the build space is created. - - REQ-EC19. - """ + """Check that the tmp_dir is reset when the build space is created.""" bs = BuildSpace(root_dir=os.getcwd()) marker = os.path.join(bs.subdir(name="tmp"), "deleteme") mkdir(bs.tmp_dir) diff --git a/tests/tests_e3/conftest.py b/tests/tests_e3/conftest.py index 5474b9ef..a7f7d553 100644 --- a/tests/tests_e3/conftest.py +++ b/tests/tests_e3/conftest.py @@ -1,20 +1,13 @@ -# type: ignore from __future__ import annotations -from os import environ from os.path import abspath, dirname, join as path_join, isfile, isdir from functools import partial -from tempfile import mkdtemp -from yaml import safe_dump as yaml_safe_dump from json import loads as json_loads from typing import TYPE_CHECKING from re import compile as regex_compile from traceback import format_stack as traceback_format_stack -from e3.env import Env -from e3.fs import rm, mkdir -from e3.os.fs import cd, which -from e3.config import Config +from e3.fs import mkdir from e3.python.wheel import Wheel import pytest @@ -22,158 +15,11 @@ if TYPE_CHECKING: from typing import Any -# When the variable RESULTS_DIR is set to -# an existing directory, the testsuite will -# generate results file in "anod" format -RESULTS_DIR = environ.get("RESULTS_DIR") +from e3.pytest import require_tool -class RequirementCoverage: - """Track requirements <-> tests.""" - - output_filename = None - results = {} - - @classmethod - def dump(cls): - if cls.output_filename: - with open(cls.output_filename, "w") as f: - yaml_safe_dump(cls.results, f) - - -@pytest.fixture(autouse=True) -def env_protect(request): - """Protection against environment change. - - The fixture is enabled for all tests and does the following: - - * store/restore env between each tests - * create a temporary directory and do a cd to it before each - test. The directory is automatically removed when test ends - """ - Env().store() - tempd = mkdtemp() - cd(tempd) - Config.data = {} - - def restore_env(): - Env().restore() - rm(tempd, True) - - request.addfinalizer(restore_env) - - -def pytest_configure(config): - try: - RequirementCoverage.output_filename = config.getoption( - "requirement_coverage_report" - ) - except ValueError: - # Option not defined. - pass - - # Define this attribute to detect errors (not failures!) in tests. - # This allows us to return a custom exit code, to differentiate between - # test *failures* and actual *errors* (like syntactic errors) in the test - # files themselves. - pytest.test_errors = False - - -def require_vcs(prog, request): - """Require svn or git to run the test. - - When in "CI" mode, a missing svn or git generates an error. In other - modes the test is just skipped. - :param prog: either "svn" or "git" - """ - if not which(prog): - if request.config.getoption("ci"): - pytest.fail(f"{prog} not available") - else: - pytest.skip(f"{prog} not available") - - -@pytest.fixture(autouse=True) -def require_git(request): - """Require git.""" - marker = request.node.get_closest_marker("git") - if marker: - return require_vcs("git", request) - - -@pytest.fixture -def git(request): - """Require git.""" - return require_vcs("git", request) - - -@pytest.fixture(autouse=True) -def require_svn(request): - """Require svn.""" - marker = request.node.get_closest_marker("svn") - if marker: - return require_vcs("svn", request) - - -@pytest.fixture -def svn(request): - """Require svn.""" - return require_vcs("svn", request) - - -def pytest_itemcollected(item): - """Keep track of all test linked to a requirement.""" - doc = item.obj.__doc__ - if RequirementCoverage.output_filename and doc: - for line in item.obj.__doc__.splitlines(): - line = line.strip().strip(".") - if line.startswith("REQ-"): - RequirementCoverage.results[item.obj.__name__] = line - - -def pytest_collectreport(report): - """Output requirement coverage report.""" - RequirementCoverage.dump() - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - """Generate results file that can be used by anod.""" - # execute all other hooks to obtain the report object - outcome = yield - rep = outcome.get_result() - - if not RESULTS_DIR or not isdir(RESULTS_DIR): - return - - # we only look at actual test calls, not setup/teardown - if rep.when == "call": - outcome = rep.outcome.upper() - test_name = rep.nodeid.replace("/", ".").replace("::", "--") - if rep.longreprtext: - with open(path_join(RESULTS_DIR, f"{test_name}.diff"), "w") as f: - f.write(rep.longreprtext) - - with open(path_join(RESULTS_DIR, "results"), "a") as f: - f.write(f"{test_name}:{outcome}\n") - else: - # If we detect a failure in an item that is not a "proper" test call, it's most - # likely an error. - # For example, this could be a failing assertion or a syntax error in a - # setup/teardown context. - if rep.outcome == "failed": - pytest.test_errors = True - - -@pytest.hookimpl(trylast=True) -def pytest_sessionfinish(session, exitstatus): - """Manage the exit code depending on if errors were detected or not.""" - if pytest.test_errors: - # Return with an exit code of `3` if we encountered errors (not failures). - # This is the exit code that corresponds to an "internal error" according to the - # pytest docs, which is the closest thing to having an actual Python error in - # test code. - session.exitstatus = 3 +git = require_tool("git") +svn = require_tool("svn") class PypiSimulator: @@ -182,7 +28,7 @@ class PypiSimulator: DATA_DIR = path_join(dirname(abspath(__file__)), "pypi_data") def __init__(self, requests_mock: Any) -> None: - self.mocked_download_urls = set() + self.mocked_download_urls: set[str] = set() self.requests_mock = requests_mock self.requests_mock.stop() @@ -274,7 +120,7 @@ def __enter__(self): self.requests_mock.get(self.MATCHER, json=self.get_metadata) return self - def __exit__(self, type_t: Any, value: Any, traceback: Any): + def __exit__(self, type_t: Any, value: Any, traceback: Any) -> None: self.requests_mock.stop() diff --git a/tests/tests_e3/vcs/git/main_test.py b/tests/tests_e3/vcs/git/main_test.py index 4df562ce..7c1f1038 100644 --- a/tests/tests_e3/vcs/git/main_test.py +++ b/tests/tests_e3/vcs/git/main_test.py @@ -10,8 +10,7 @@ from contextlib import closing -@pytest.mark.git -def test_git_non_utf8(): +def test_git_non_utf8(git): """Test with non utf-8 encoding in changelog.""" working_tree = os.path.join(os.getcwd(), "working_tree") repo = GitRepository(working_tree) @@ -43,8 +42,7 @@ def test_git_non_utf8(): assert "\\x03\\xff" in commits[0]["diff"] -@pytest.mark.git -def test_git_repo(): +def test_git_repo(git): working_tree = os.path.join(os.getcwd(), "working_tree") working_tree2 = os.path.join(os.getcwd(), "working_tree2") repo = GitRepository(working_tree) diff --git a/tests/tests_e3/vcs/svn/main_test.py b/tests/tests_e3/vcs/svn/main_test.py index 3f16b574..3d24983b 100644 --- a/tests/tests_e3/vcs/svn/main_test.py +++ b/tests/tests_e3/vcs/svn/main_test.py @@ -22,8 +22,7 @@ def file_url(path, unix=False): return "file://" + path -@pytest.mark.svn -def test_svn_repo(): +def test_svn_repo(svn): cwd = os.getcwd() # --- create local project diff --git a/tox.ini b/tox.ini index d53e33dd..b0e29cae 100644 --- a/tox.ini +++ b/tox.ini @@ -15,17 +15,11 @@ extras = # Run testsuite with coverage when '-cov' is in the env name commands= - {envpython} {toxinidir}/tests/gen-cov-config.py {toxinidir}/.coveragerc pytest --ignore=build -vv --html=pytest-report.html --self-contained-html \ xdist: -n auto \ - --disable-socket \ - ci: --ci \ + cov: --e3-cov-rewrite {envsitepackagesdir} src \ cov: --cov={envsitepackagesdir}/e3 --cov-report= --cov-fail-under=0 --cov-branch \ [] - cov: {envpython} {toxinidir}/tests/fix-coverage-paths.py \ - cov: {envsitepackagesdir} {toxinidir}/.coverage - cov: coverage html --fail-under=0 - cov: coverage report [testenv:check] # Run mypy, pip audit, and bandit @@ -44,25 +38,12 @@ commands = [testenv:docs] deps = - pytest - mock - httpretty sphinx sphinx-autoapi sphinx_rtd_theme commands = - pytest --collect-only --requirement-coverage-report={toxinidir}/docs/source/requirement_coverage.yaml - python docs/generate-req-coverage.py {toxinidir}/docs/source/requirements.yaml \ - {toxinidir}/docs/source/requirement_coverage.yaml \ - {toxinidir}/docs/source/requirement_coverage.rst python -msphinx -M html {toxinidir}/docs/source {toxinidir}/docs/build -[pytest] -addopts = --failed-first -markers = - git: git needs to be installed to run these tests - svn: svn needs to be installed to run these tests - [flake8] exclude = .git,__pycache__,build,dist,.tox ignore = A003, C901, E203, E266, E501, W503,D100,D101,D102,D102,D103,D104,D105,D106,D107,D203,D403,D213,B028,B906,B907,E704 From 9bdc6e7b6b9b56c1cd1e47c9fdd33ee98dbf9146 Mon Sep 17 00:00:00 2001 From: Olivier Ramonat Date: Tue, 26 Mar 2024 10:00:59 +0100 Subject: [PATCH 2/3] Fix release dates --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 0568eaae..1de52192 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,9 @@ -# Version 22.5.0 (2023-??-??) *NOT RELEASED YET* +# Version 22.5.0 (2024-??-??) *NOT RELEASED YET* * Add e3.pytest plugin to reuse fixtures in other projects -# Version 22.4.0 (2023-01-18) +# Version 22.4.0 (2024-01-18) * Security enhancements: * e3.net.smtp.sendmail uses to ``SMTP_SSL`` by default From d8fa567cd134947ec46945fd7cb0787e6b23d5ad Mon Sep 17 00:00:00 2001 From: Olivier Ramonat Date: Tue, 26 Mar 2024 13:57:54 +0100 Subject: [PATCH 3/3] Automatically disable pytest-cov report when using e3 pytest plugin When using e3 pytest plugin, some coverage options are set. For instance we ignore some lines in the code source, or omit some files. The pytest-cov report is not accurate and is automatically disabled in that case. --- src/e3/pytest.py | 12 ++++++++++++ tox.ini | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/e3/pytest.py b/src/e3/pytest.py index 3d801f4c..574b1fe2 100644 --- a/src/e3/pytest.py +++ b/src/e3/pytest.py @@ -97,11 +97,23 @@ def restore_env() -> None: request.addfinalizer(restore_env) +def pytest_configure(config: pytest.Config) -> None: + if config.getoption("e3") and config.getoption("cov_source"): + # When e3 plugin is activated, the report generated by pytest-cov + # should be deactivated to avoid duplicating the output. Also + # some options set by e3 change the coverage, pytest-cov report is not + # accurate in that case. + cov = config.pluginmanager.getplugin("_cov") + cov.options.cov_report = {} + cov.options.cov_fail_under = None + + @pytest.hookimpl(trylast=True) def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: """Manage the exit code depending on if errors were detected or not.""" if not session.config.getoption("e3"): return + global test_errors if test_errors: # Return with an exit code of `3` if we encountered errors (not failures). diff --git a/tox.ini b/tox.ini index b0e29cae..bbf83f2a 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ commands= pytest --ignore=build -vv --html=pytest-report.html --self-contained-html \ xdist: -n auto \ cov: --e3-cov-rewrite {envsitepackagesdir} src \ - cov: --cov={envsitepackagesdir}/e3 --cov-report= --cov-fail-under=0 --cov-branch \ + cov: --cov={envsitepackagesdir}/e3 --cov-branch \ [] [testenv:check]