Get rid of numpy.distutils

.github/workflows/core.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.9, '3.11', pypy-3.9]
+        python-version: [3.7, 3.9, 3.11, 3.12, pypy-3.9]
         cpp-version: [g++-8, clang-7]
     steps:
     - uses: actions/checkout@v2
@@ -24,7 +24,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install ipython nbval pytest-xdist cython wheel
+        pip install -r pythran/tests/requirements.txt
+        pip install pytest-xdist
         sudo apt install libopenblas-dev ${{ matrix.cpp-version }}
         if test ${{ matrix.python-version }} != 'pypy-3.9'; then pip install scipy ; fi
     - name: Setup

.github/workflows/doc.yml

@@ -24,7 +24,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install ipython nbval pytest-xdist cython scipy wheel
+        pip install -r pythran/tests/requirements.txt scipy
+        pip install pytest-xdist
         sudo apt install libopenblas-dev ${{ matrix.cpp-version }}
     - name: Setup
       run: |

.github/workflows/icc.yml

@@ -24,7 +24,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install ipython nbval pytest-xdist cython wheel
+        pip install -r pythran/tests/requirements.txt
+        pip install pytest-xdist
         if test ${{ matrix.python-version }} != '3.10' ; then pip install scipy ; fi
         wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg
         echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list

.github/workflows/parallel.yml

@@ -24,7 +24,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install ipython nbval pytest-xdist cython scipy wheel
+        pip install -r pythran/tests/requirements.txt scipy
+        pip install pytest-xdist
         sudo apt install libopenblas-dev ${{ matrix.cpp-version }}
     - name: Setup
       run: |

.github/workflows/windows.yml

@@ -28,7 +28,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install scipy wheel pythran-openblas pytest
+        pip install -r pythran/tests/requirements.txt scipy
+        pip install pythran-openblas pytest
     - name: Setup
       run: |
         python setup.py install

docs/DEVGUIDE.rst

@@ -68,8 +68,12 @@ Validation
 ----------
 
 ``pythran`` uses the ``unittest`` module and the `pytest
-<http://pytest.org/latest/>`_ package to manage test cases. The whole
-validation suite is run through the command::
+<http://pytest.org/latest/>`_ package to manage test cases.
+
+All requirements are listed in ``pythran/tests/requirements.txt``.
+
+
+The whole validation suite is run through the command::
 
     $> python -m pytest pythran/tests
 

pythran/config.py

@@ -246,26 +246,20 @@ def parse_define(define):
         extension['define_macros'].append('PYTHRAN_BLAS_NONE')
 
     if user_blas not in reserved_blas_entries:
-        if sys.version_info < (3, 12):
-            # `numpy.distutils` not present for Python >= 3.12
-            try:
-                import numpy.distutils.system_info as numpy_sys
-                 # Numpy can pollute stdout with checks
-                with silent():
-                    numpy_blas = numpy_sys.get_info(user_blas)
-                    # required to cope with atlas missing extern "C"
-                    extension['define_macros'].append('PYTHRAN_BLAS_{}'
-                                                      .format(user_blas.upper()))
-                    extension['libraries'].extend(numpy_blas.get('libraries', []))
-                    extension['library_dirs'].extend(
-                        numpy_blas.get('library_dirs', []))
-                    extension['include_dirs'].extend(
-                        numpy_blas.get('include_dirs', []))
-            except Exception as exc:
-                raise RuntimeError(
-                    "The likely cause of this failure is an incompatibility "
-                    "between `setuptools` and `numpy.distutils. "
-                ) from exc
+        try:
+            import numpy.distutils.system_info as numpy_sys
+             # Numpy can pollute stdout with checks
+            with silent():
+                numpy_blas = numpy_sys.get_info(user_blas)
+                extension['libraries'].extend(numpy_blas.get('libraries', []))
+                extension['library_dirs'].extend(
+                    numpy_blas.get('library_dirs', []))
+        # `numpy.distutils` not present for Python >= 3.12
+        except ImportError:
+            blas = numpy.show_config('dicts')["Build Dependencies"]["blas"]
+            libblas = {'openblas64': 'openblas'}.get(blas['name'], blas['name'])
+            extension["libraries"].append(libblas)
+
 
     # final macro normalization
     extension["define_macros"] = [

pythran/dist.py

@@ -13,14 +13,17 @@
 import os.path
 import os
 
-from distutils.command.build_ext import build_ext as LegacyBuildExt
+try:
+    from distutils.command.build_ext import build_ext as LegacyBuildExt
+except ImportError:
+    from setuptools.command.build_ext import build_ext as LegacyBuildExt
 
 try:
     # `numpy.distutils` is deprecated, and won't be present on Python >=3.12
     # If it is installed, we need to use it though, so try-import it:
     from numpy.distutils.extension import Extension
 except ImportError:
-    from distutils.extension import Extension
+    from setuptools.extension import Extension
 
 
 

pythran/tests/requirements.txt

@@ -0,0 +1,4 @@
+ipython
+nbval
+cython
+wheel

pythran/tests/test_distutils.py

@@ -29,7 +29,10 @@ class TestDistutils(unittest.TestCase):
     def test_setup_build(self):
         check_call([python, 'setup.py', 'build'],
                    cwd=os.path.join(cwd, 'test_distutils'))
-        check_call([python, '-m', 'pip', 'install', '.', '--prefix=demo_install'],
+
+        extra_args = ['--no-build-isolation',]
+        check_call([python, '-m', 'pip', 'install', '.',
+                    '--prefix=demo_install'] + extra_args,
                    cwd=os.path.join(cwd, 'test_distutils'))
 
         base = os.path.join(cwd, 'test_distutils', 'demo_install',)
@@ -81,7 +84,10 @@ def test_setup_wheel_install(self):
     def test_setup_build2(self):
         check_call([python, 'setup.py', 'build'],
                    cwd=os.path.join(cwd, 'test_distutils_packaged'))
-        check_call([python, '-m', 'pip', 'install', '.', '--prefix=demo_install2'],
+
+        extra_args = ['--no-build-isolation',]
+        check_call([python, '-m', 'pip', 'install', '.',
+                    '--prefix=demo_install2'] + extra_args,
                    cwd=os.path.join(cwd, 'test_distutils_packaged'))
 
         base = os.path.join(cwd, 'test_distutils_packaged', 'demo_install2',)
@@ -116,19 +122,30 @@ def test_setup_bdist_install2(self):
         self.assertIsNotNone(demo_so)
         shutil.rmtree(dist_path)
 
+    @unittest.skipIf(sys.version_info >= (3, 12), "setup install is deprecated")
     def test_setup_build3(self):
         check_call([python, 'setup.py', 'build'],
                    cwd=os.path.join(cwd, 'test_distutils_numpy'))
-        check_call([python, 'setup.py', 'install', '--prefix=demo_install3'],
-                   cwd=os.path.join(cwd, 'test_distutils_numpy'))
 
         base = os.path.join(cwd, 'test_distutils_numpy', 'demo_install3',)
         libdir = os.path.join(base, 'lib')
-        if not os.path.isdir(libdir):
-            libdir = os.path.join(base, 'lib64')
-        check_call([python, '-c', 'import a'],
-                   cwd=os.path.join(libdir, python_version, 'site-packages',
-                                    'demo3'))
+        lib64dir = os.path.join(base, 'lib64')
+
+        local_env = os.environ.copy()
+        local_env['PYTHONPATH'] = os.pathsep.join((
+            os.path.join(lib64dir, python_version,
+                         'site-packages'),
+            os.path.join(libdir, python_version,
+                         'site-packages'),
+            local_env.get('PYTHONPATH', ''),
+            ))
+
+        check_call([python, 'setup.py', 'install', '--prefix=demo_install3'],
+                   cwd=os.path.join(cwd, 'test_distutils_numpy'),
+                   env=local_env)
+
+        check_call([python, '-c', 'import demo3.a'],
+                   cwd=cwd, env=local_env)
         check_call([python, 'setup.py', 'clean'],
                    cwd=os.path.join(cwd, 'test_distutils_numpy'))
         shutil.rmtree(os.path.join(cwd, 'test_distutils_numpy', 'demo_install3'))

pythran/tests/test_distutils_numpy/setup.py

@@ -1,5 +1,10 @@
-from numpy.distutils.core import setup
-from numpy.distutils.command.build_ext import build_ext as npy_build_ext
+try:
+    from numpy.distutils.core import setup
+    from numpy.distutils.command.build_ext import build_ext as npy_build_ext
+except ImportError:
+    from distutils.core import setup
+    from distutils.command.build_ext import build_ext as npy_build_ext
+
 from pythran.dist import PythranExtension, PythranBuildExt
 
 module1 = PythranExtension('demo3.a', sources = ['demo3/a.py'])

pythran/toolchain.py

@@ -22,13 +22,17 @@
 from pythran.utils import cxxid
 import pythran.frontend as frontend
 
-from distutils.errors import CompileError
-from distutils import sysconfig
+try:
+    from distutils.errors import CompileError
+    from distutils import sysconfig
+except ImportError:
+    from setuptools.errors import CompileError
+    from setuptools._distutils import sysconfig
 try:
     # `numpy.distutils is deprecated, may not be present, or broken
     from numpy.distutils.core import setup
 except Exception:
-    from distutils.core import setup
+    from setuptools import setup
 
 from tempfile import mkdtemp, NamedTemporaryFile
 import gast as ast

requirements.txt

@@ -1,4 +1,5 @@
 ply>=3.4
+setuptools
 gast~=0.5.0
 numpy
 beniget~=0.4.0