Merge branch 'topic/default/ci-py3.12' into 'branch/default'

CI Github with 3.12 + better noxfile + coverage xml + fix Numba issue with Meson

See merge request fluiddyn/transonic!129

.github/workflows/ci-linux.yml

@@ -9,12 +9,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - name: apt install
       run: |
-        sudo apt install -y libopenmpi-dev
+        sudo apt install -y libopenmpi-dev libopenblas-dev
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
@@ -23,12 +23,33 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pdm nox
-    - name: Test with nox
+        pip install pdm nox coverage
+    - name: Test without Pythran
       run: |
-        nox -s test_without_pythran test_with_pythran
+        rm -rf .coverage
+        nox -s "test(with_pythran=0, with_cython=0)"
+        coverage combine
+        coverage report
+        coverage xml
+        mv .coverage/coverage.xml coverage_without_pythran.xml
+    - name: Test with Pythran
+      run: |
+        rm -rf .coverage
+        nox -s "test(with_pythran=1, with_cython=0)"
+        coverage combine
+        coverage report
+        coverage xml
+        mv .coverage/coverage.xml coverage_with_pythran.xml
+    - name: Test with Cython
+      run: |
+        rm -rf .coverage
+        nox -s "test(with_pythran=0, with_cython=1)"
+        coverage combine
+        coverage xml
+        mv .coverage/coverage.xml coverage_with_cython.xml
     - uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         fail_ci_if_error: false # optional (default = false)
         verbose: true # optional (default = false)
+        files: coverage_without_pythran.xml,coverage_with_pythran.xml,coverage_with_cython.xml

.gitlab-ci.yml

@@ -75,23 +75,31 @@ step_without_pythran:
     - job: "image:build"
       optional: true
   script:
-    - nox -s test_without_pythran
+    - nox -s "test(with_pythran=0, with_cython=0)"
 
-step_pythran_then_cython:
+step_with_pythran:
   stage: test
   needs:
     - job: "image:build"
       optional: true
   script:
-    - nox -s test_with_pythran test_with_cython
+    - nox -s "test(with_pythran=1, with_cython=0)"
 
-step_pythran_cython:
+step_with_cython:
   stage: test
   needs:
     - job: "image:build"
       optional: true
   script:
-    - nox -s test_with_pythran_cython
+    - nox -s "test(with_pythran=0, with_cython=1)"
+
+step_with_pythran_cython:
+  stage: test
+  needs:
+    - job: "image:build"
+      optional: true
+  script:
+    - nox -s "test(with_pythran=1, with_cython=1)"
 
 
 pages:

README.md

@@ -5,7 +5,7 @@
 [![Documentation status](https://readthedocs.org/projects/transonic/badge/?version=latest)](http://transonic.readthedocs.org)
 ![Supported Python versions](https://img.shields.io/pypi/pyversions/transonic.svg)
 [![Heptapod CI](https://foss.heptapod.net/fluiddyn/transonic/badges/branch/default/pipeline.svg)](https://foss.heptapod.net/fluiddyn/transonic/-/pipelines)
-[![Github Actions](https://github.com/fluiddyn/transonic/actions/workflows/ci.yml/badge.svg?branch=branch/default)](https://github.com/fluiddyn/transonic/actions)
+[![Github Actions](https://github.com/fluiddyn/transonic/actions/workflows/ci-linux.yml/badge.svg?branch=branch/default)](https://github.com/fluiddyn/transonic/actions)
 [![mybinder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fluiddyn/transonic/branch/default?urlpath=lab/tree/doc/ipynb/executed)
 [![sonarcloud](https://sonarcloud.io/api/project_badges/measure?project=fluiddyn_transonic&metric=alert_status)](https://sonarcloud.io/dashboard?id=fluiddyn_transonic)
 

data_tests/package_for_test_meson/pyproject.toml

@@ -13,4 +13,4 @@ license = {text = "BSD License"}
 [build-system]
 build-backend = "mesonpy"
 # We cannot add the local transonic as build dependency here
-requires = ["meson-python", "transonic"]
+requires = ["meson-python", "transonic", "pythran"]

noxfile.py

@@ -1,48 +1,69 @@
 import os
 import sys
 from packaging import version
+from pathlib import Path
 
 import nox
 
 os.environ.update({"PDM_IGNORE_SAVED_PYTHON": "1"})
+nox.options.reuse_existing_virtualenvs = 1
 
 
-def _test(session):
-    session.run("make", "tests_ipynb", external=True)
-    session.run("make", "tests_coverage", external=True)
-
-
-def _install_base(session):
+@nox.parametrize("with_cython", [0, 1])
+@nox.parametrize("with_pythran", [0, 1])
+@nox.session
+def test(session, with_pythran, with_cython):
     command = "pdm sync -G base_test"
     session.run_always(*command.split(), external=True)
 
-    py_version = session.python if session.python is not None else sys.version.split(maxsplit=1)[0]
+    py_version = (
+        session.python
+        if session.python is not None
+        else sys.version.split(maxsplit=1)[0]
+    )
     if version.parse(py_version) < version.parse("3.12"):
         session.install("numba")
+    else:
+        session.install("setuptools")
 
+    if with_pythran:
+        session.install("pythran")
+    if with_cython:
+        session.install("cython")
 
-@nox.session
-def test_without_pythran(session):
-    _install_base(session)
-    _test(session)
-
-
-@nox.session
-def test_with_pythran(session):
-    _install_base(session)
-    session.install("pythran")
-    _test(session)
+    if version.parse(py_version) < version.parse("3.12"):
+        for backend in ("python", "pythran"):
+            print(f"TRANSONIC_BACKEND={backend}")
+            session.run(
+                "pytest",
+                "--nbval-lax",
+                "data_tests/ipynb",
+                env={"TRANSONIC_BACKEND": backend},
+            )
 
+    path_coverage = Path(".coverage")
+    path_coverage.mkdir(exist_ok=True)
 
-@nox.session
-def test_with_cython(session):
-    _install_base(session)
-    session.install("cython")
-    _test(session)
+    code_dependencies = 10 * with_pythran + with_cython
 
+    for backend in ("python", "pythran", "numba", "cython"):
+        print(f"TRANSONIC_BACKEND={backend}")
+        session.run(
+            "pytest",
+            "--cov",
+            "--cov-config=pyproject.toml",
+            "tests",
+            env={
+                "COVERAGE_FILE": f".coverage/coverage{code_dependencies}.{backend}",
+                "TRANSONIC_BACKEND": backend,
+            },
+        )
 
-@nox.session
-def test_with_pythran_cython(session):
-    _install_base(session)
-    session.install("pythran", "cython")
-    _test(session)
+    command = "mpirun -np 2 coverage run --rcfile=pyproject.toml -m mpi4py -m pytest tests"
+    session.run(
+        *command.split(),
+        external=True,
+        env={
+            "TRANSONIC_BACKEND": "pythran",
+        },
+    )

pyproject.toml

@@ -81,7 +81,7 @@ target-version = ['py38']
 [tool.coverage.run]
 branch = true
 
-source = ["src/transonic", "src/transonic_cl", "./tests"]
+source = ["./src/transonic", "./src/transonic_cl", "./tests"]
 data_file = ".coverage/coverage"
 omit = [
     "*/try_*.py",

src/transonic/backends/base.py

@@ -64,7 +64,7 @@ def _make_code_from_fdef_node(self, fdef):
         return format_str(code)
 
     def make_backend_files(
-        self, paths_py, force=False, log_level=None, analyses=None
+        self, paths_py, force=False, log_level=None, analyses=None, **kwargs
     ):
         """Create backend files from a list of Python files"""
 
@@ -78,7 +78,9 @@ def make_backend_files(
         paths_out = []
         for path in paths_py:
             analysis = analyses[path]
-            path_out = self.make_backend_file(path, analysis, force=force)
+            path_out = self.make_backend_file(
+                path, analysis, force=force, **kwargs
+            )
             if path_out:
                 paths_out.append(path_out)
 
@@ -98,7 +100,7 @@ def make_backend_files(
         return paths_out
 
     def make_backend_file(
-        self, path_py: Path, analysis=None, force=False, log_level=None
+        self, path_py: Path, analysis=None, force=False, log_level=None, **kwargs
     ):
         """Create a Python file from a Python file (if necessary)"""
 
@@ -134,7 +136,7 @@ def make_backend_file(
             analysis = analyse_aot(code, path_py)
 
         code_backend, codes_ext, code_header = self._make_backend_code(
-            path_py, analysis
+            path_py, analysis, **kwargs
         )
         if not code_backend:
             return
@@ -178,7 +180,7 @@ def _make_first_lines_header(self):
     def _make_beginning_code(self):
         return ""
 
-    def _make_backend_code(self, path_py, analysis):
+    def _make_backend_code(self, path_py, analysis, **kwargs):
         """Create a backend code from a Python file"""
 
         boosted_dicts, code_dependance, annotations, blocks, codes_ext = analysis

src/transonic/backends/numba.py

@@ -82,11 +82,17 @@ def compile_extension(
         process = None
         return compiling, process
 
-    def _make_backend_code(self, path_py, analysis):
+    def _make_backend_code(self, path_py, analysis, **kwargs):
         """Create a backend code from a Python file"""
         code, codes_ext, header = super()._make_backend_code(path_py, analysis)
 
         if not code:
             return code, codes_ext, header
 
-        return add_numba_comments(code), codes_ext, header
+        code = add_numba_comments(code)
+
+        for_meson = kwargs.get("for_meson", False)
+        if for_meson:
+            code = format_str(code.replace("# __protected__ ", ""))
+
+        return code, codes_ext, header

src/transonic/run.py

@@ -95,7 +95,9 @@ def run():
 
 
 def run_1_backend(paths, backend, args, analyses):
-    backend.make_backend_files(paths, force=args.force, analyses=analyses)
+    backend.make_backend_files(
+        paths, force=args.force, analyses=analyses, for_meson=args.meson
+    )
 
     if args.meson:
         path_meson_build = Path("meson.build")