diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index 6241949..255c6e3 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -9,12 +9,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: apt install run: | - sudo apt install -y libopenmpi-dev + sudo apt install -y libopenmpi-dev libopenblas-dev - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -23,12 +23,33 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pdm nox - - name: Test with nox + pip install pdm nox coverage + - name: Test without Pythran run: | - nox -s test_without_pythran test_with_pythran + rm -rf .coverage + nox -s "test(with_pythran=0, with_cython=0)" + coverage combine + coverage report + coverage xml + mv .coverage/coverage.xml coverage_without_pythran.xml + - name: Test with Pythran + run: | + rm -rf .coverage + nox -s "test(with_pythran=1, with_cython=0)" + coverage combine + coverage report + coverage xml + mv .coverage/coverage.xml coverage_with_pythran.xml + - name: Test with Cython + run: | + rm -rf .coverage + nox -s "test(with_pythran=0, with_cython=1)" + coverage combine + coverage xml + mv .coverage/coverage.xml coverage_with_cython.xml - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false # optional (default = false) verbose: true # optional (default = false) + files: coverage_without_pythran.xml,coverage_with_pythran.xml,coverage_with_cython.xml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b3fde7..4a9ee58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,23 +75,31 @@ step_without_pythran: - job: "image:build" optional: true script: - - nox -s test_without_pythran + - nox -s "test(with_pythran=0, with_cython=0)" -step_pythran_then_cython: +step_with_pythran: stage: test needs: - job: "image:build" optional: true script: - - nox -s test_with_pythran test_with_cython + - nox -s "test(with_pythran=1, with_cython=0)" -step_pythran_cython: +step_with_cython: stage: test needs: - job: "image:build" optional: true script: - - nox -s test_with_pythran_cython + - nox -s "test(with_pythran=0, with_cython=1)" + +step_with_pythran_cython: + stage: test + needs: + - job: "image:build" + optional: true + script: + - nox -s "test(with_pythran=1, with_cython=1)" pages: diff --git a/README.md b/README.md index 706bd6a..2d18b61 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Documentation status](https://readthedocs.org/projects/transonic/badge/?version=latest)](http://transonic.readthedocs.org) ![Supported Python versions](https://img.shields.io/pypi/pyversions/transonic.svg) [![Heptapod CI](https://foss.heptapod.net/fluiddyn/transonic/badges/branch/default/pipeline.svg)](https://foss.heptapod.net/fluiddyn/transonic/-/pipelines) -[![Github Actions](https://github.com/fluiddyn/transonic/actions/workflows/ci.yml/badge.svg?branch=branch/default)](https://github.com/fluiddyn/transonic/actions) +[![Github Actions](https://github.com/fluiddyn/transonic/actions/workflows/ci-linux.yml/badge.svg?branch=branch/default)](https://github.com/fluiddyn/transonic/actions) [![mybinder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fluiddyn/transonic/branch/default?urlpath=lab/tree/doc/ipynb/executed) [![sonarcloud](https://sonarcloud.io/api/project_badges/measure?project=fluiddyn_transonic&metric=alert_status)](https://sonarcloud.io/dashboard?id=fluiddyn_transonic) diff --git a/data_tests/package_for_test_meson/pyproject.toml b/data_tests/package_for_test_meson/pyproject.toml index 1da8173..0c9e081 100644 --- a/data_tests/package_for_test_meson/pyproject.toml +++ b/data_tests/package_for_test_meson/pyproject.toml @@ -13,4 +13,4 @@ license = {text = "BSD License"} [build-system] build-backend = "mesonpy" # We cannot add the local transonic as build dependency here -requires = ["meson-python", "transonic"] +requires = ["meson-python", "transonic", "pythran"] diff --git a/noxfile.py b/noxfile.py index a2b44e3..eb1f8a6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,48 +1,69 @@ import os import sys from packaging import version +from pathlib import Path import nox os.environ.update({"PDM_IGNORE_SAVED_PYTHON": "1"}) +nox.options.reuse_existing_virtualenvs = 1 -def _test(session): - session.run("make", "tests_ipynb", external=True) - session.run("make", "tests_coverage", external=True) - - -def _install_base(session): +@nox.parametrize("with_cython", [0, 1]) +@nox.parametrize("with_pythran", [0, 1]) +@nox.session +def test(session, with_pythran, with_cython): command = "pdm sync -G base_test" session.run_always(*command.split(), external=True) - py_version = session.python if session.python is not None else sys.version.split(maxsplit=1)[0] + py_version = ( + session.python + if session.python is not None + else sys.version.split(maxsplit=1)[0] + ) if version.parse(py_version) < version.parse("3.12"): session.install("numba") + else: + session.install("setuptools") + if with_pythran: + session.install("pythran") + if with_cython: + session.install("cython") -@nox.session -def test_without_pythran(session): - _install_base(session) - _test(session) - - -@nox.session -def test_with_pythran(session): - _install_base(session) - session.install("pythran") - _test(session) + if version.parse(py_version) < version.parse("3.12"): + for backend in ("python", "pythran"): + print(f"TRANSONIC_BACKEND={backend}") + session.run( + "pytest", + "--nbval-lax", + "data_tests/ipynb", + env={"TRANSONIC_BACKEND": backend}, + ) + path_coverage = Path(".coverage") + path_coverage.mkdir(exist_ok=True) -@nox.session -def test_with_cython(session): - _install_base(session) - session.install("cython") - _test(session) + code_dependencies = 10 * with_pythran + with_cython + for backend in ("python", "pythran", "numba", "cython"): + print(f"TRANSONIC_BACKEND={backend}") + session.run( + "pytest", + "--cov", + "--cov-config=pyproject.toml", + "tests", + env={ + "COVERAGE_FILE": f".coverage/coverage{code_dependencies}.{backend}", + "TRANSONIC_BACKEND": backend, + }, + ) -@nox.session -def test_with_pythran_cython(session): - _install_base(session) - session.install("pythran", "cython") - _test(session) + command = "mpirun -np 2 coverage run --rcfile=pyproject.toml -m mpi4py -m pytest tests" + session.run( + *command.split(), + external=True, + env={ + "TRANSONIC_BACKEND": "pythran", + }, + ) diff --git a/pdm.lock b/pdm.lock index 3bea342..ed78f25 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "base_test", "dev", "doc", "mpi", "test"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:2ee032cde30f51d683cf1c67788207359de6de2689d3082e3ee48da9b7263784" +content_hash = "sha256:947d7510c400794a3ee19d3303734c783a19ced37af5e0a08260056d77e79104" [[package]] name = "alabaster" @@ -896,7 +896,7 @@ files = [ [[package]] name = "ipykernel" -version = "6.28.0" +version = "6.29.0" requires_python = ">=3.8" summary = "IPython Kernel for Jupyter" groups = ["base_test", "doc"] @@ -916,8 +916,8 @@ dependencies = [ "traitlets>=5.4.0", ] files = [ - {file = "ipykernel-6.28.0-py3-none-any.whl", hash = "sha256:c6e9a9c63a7f4095c0a22a79f765f079f9ec7be4f2430a898ddea889e8665661"}, - {file = "ipykernel-6.28.0.tar.gz", hash = "sha256:69c11403d26de69df02225916f916b37ea4b9af417da0a8c827f84328d88e5f3"}, + {file = "ipykernel-6.29.0-py3-none-any.whl", hash = "sha256:076663ca68492576f051e4af7720d33f34383e655f2be0d544c8b1c9de915b2f"}, + {file = "ipykernel-6.29.0.tar.gz", hash = "sha256:b5dd3013cab7b330df712891c96cd1ab868c27a7159e606f762015e9bf8ceb3f"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index cbb92f0..bfd2067 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ target-version = ['py38'] [tool.coverage.run] branch = true -source = ["src/transonic", "src/transonic_cl", "./tests"] +source = ["./src/transonic", "./src/transonic_cl", "./tests"] data_file = ".coverage/coverage" omit = [ "*/try_*.py", diff --git a/src/transonic/backends/base.py b/src/transonic/backends/base.py index 1e73fe0..0544ab0 100644 --- a/src/transonic/backends/base.py +++ b/src/transonic/backends/base.py @@ -64,7 +64,7 @@ def _make_code_from_fdef_node(self, fdef): return format_str(code) def make_backend_files( - self, paths_py, force=False, log_level=None, analyses=None + self, paths_py, force=False, log_level=None, analyses=None, **kwargs ): """Create backend files from a list of Python files""" @@ -78,7 +78,9 @@ def make_backend_files( paths_out = [] for path in paths_py: analysis = analyses[path] - path_out = self.make_backend_file(path, analysis, force=force) + path_out = self.make_backend_file( + path, analysis, force=force, **kwargs + ) if path_out: paths_out.append(path_out) @@ -98,7 +100,7 @@ def make_backend_files( return paths_out def make_backend_file( - self, path_py: Path, analysis=None, force=False, log_level=None + self, path_py: Path, analysis=None, force=False, log_level=None, **kwargs ): """Create a Python file from a Python file (if necessary)""" @@ -134,7 +136,7 @@ def make_backend_file( analysis = analyse_aot(code, path_py) code_backend, codes_ext, code_header = self._make_backend_code( - path_py, analysis + path_py, analysis, **kwargs ) if not code_backend: return @@ -178,7 +180,7 @@ def _make_first_lines_header(self): def _make_beginning_code(self): return "" - def _make_backend_code(self, path_py, analysis): + def _make_backend_code(self, path_py, analysis, **kwargs): """Create a backend code from a Python file""" boosted_dicts, code_dependance, annotations, blocks, codes_ext = analysis diff --git a/src/transonic/backends/numba.py b/src/transonic/backends/numba.py index 19f5d6e..7e2c307 100644 --- a/src/transonic/backends/numba.py +++ b/src/transonic/backends/numba.py @@ -82,11 +82,17 @@ def compile_extension( process = None return compiling, process - def _make_backend_code(self, path_py, analysis): + def _make_backend_code(self, path_py, analysis, **kwargs): """Create a backend code from a Python file""" code, codes_ext, header = super()._make_backend_code(path_py, analysis) if not code: return code, codes_ext, header - return add_numba_comments(code), codes_ext, header + code = add_numba_comments(code) + + for_meson = kwargs.get("for_meson", False) + if for_meson: + code = format_str(code.replace("# __protected__ ", "")) + + return code, codes_ext, header diff --git a/src/transonic/run.py b/src/transonic/run.py index 382d156..04e6904 100644 --- a/src/transonic/run.py +++ b/src/transonic/run.py @@ -95,7 +95,9 @@ def run(): def run_1_backend(paths, backend, args, analyses): - backend.make_backend_files(paths, force=args.force, analyses=analyses) + backend.make_backend_files( + paths, force=args.force, analyses=analyses, for_meson=args.meson + ) if args.meson: path_meson_build = Path("meson.build")