diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml index b6ff0054..b0fe9c09 100644 --- a/.github/workflows/python-wheels.yml +++ b/.github/workflows/python-wheels.yml @@ -19,7 +19,7 @@ jobs: run: pipx run build --sdist - name: Upload SDist - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: dist/*.tar.gz @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.16 + uses: pypa/cibuildwheel@v2.21 env: # skip PyPy and muslinux (for now) CIBW_SKIP: "pp* *musllinux*" @@ -60,7 +60,7 @@ jobs: if: github.event_name == 'release' && github.event.action == 'published' steps: - name: Get dist files - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: artifact path: dist diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 14597a1d..f0aaaa69 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -64,7 +64,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-20.04", "ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout repo uses: actions/checkout@v3 @@ -85,3 +85,24 @@ jobs: - name: Run tests run: | pytest python/fastscapelib/tests -vv --color=yes + + test_npy: + name: Test Python (Numpy 1.xx compat) + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build and install Fastscapelib Python (Numpy 2.x) + run: | + python -m pip install . --config-settings=cmake.define.FS_DOWNLOAD_XTENSOR_PYTHON=ON + + - name: Run tests (Numpy 1.xx) + run: | + python -m pip install numpy==1.26.4 pytest + pytest python/fastscapelib/tests -vv --color=yes diff --git a/CMakeLists.txt b/CMakeLists.txt index 90de0264..392c5d57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,14 +39,14 @@ if(SKBUILD AND FS_DOWNLOAD_XTENSOR_PYTHON) FetchContent_Declare(xtensor GIT_REPOSITORY https://github.com/xtensor-stack/xtensor - GIT_TAG 0.24.6 + GIT_TAG 0.25.0 GIT_SHALLOW TRUE) set(CPP17 ON CACHE BOOL "Enable C++17 for xtensor" FORCE) FetchContent_MakeAvailable(xtensor) FetchContent_Declare(xtensor-python GIT_REPOSITORY https://github.com/xtensor-stack/xtensor-python - GIT_TAG 0.26.1 + GIT_TAG 0.27.0 GIT_SHALLOW TRUE) FetchContent_MakeAvailable(xtensor-python) diff --git a/doc/environment.yml b/doc/environment.yml index 740e968b..fb81a31a 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -2,13 +2,14 @@ name: fastscapelib-docs channels: - conda-forge dependencies: - - python=3.9 + - python=3.11 - breathe - cmake - xtensor-python - pybind11 - scikit-build-core - numpy + - numba - matplotlib-base - pygalmesh - sphinx diff --git a/doc/source/api_python/hidden.rst b/doc/source/api_python/hidden.rst index f3ef299e..28b36a15 100644 --- a/doc/source/api_python/hidden.rst +++ b/doc/source/api_python/hidden.rst @@ -7,5 +7,5 @@ MSTMethod MSTRouteMethod - flow.numba_flow_kernel.NumbaFlowKernel - flow.numba_flow_kernel.NumbaFlowKernelData + flow.numba.flow_kernel.NumbaFlowKernel + flow.numba.flow_kernel.NumbaFlowKernelData diff --git a/doc/source/install.md b/doc/source/install.md index 45cc1e40..ff6fb992 100644 --- a/doc/source/install.md +++ b/doc/source/install.md @@ -130,7 +130,7 @@ options. ## Install the Python Library -Fastscapelib's Python package requires Python (3.9+) and [numpy]. +Fastscapelib's Python package requires Python (3.10+) and [numpy]. (install-python-binary)= diff --git a/environment-python-dev.yml b/environment-python-dev.yml index 34ec19f3..1c40b751 100644 --- a/environment-python-dev.yml +++ b/environment-python-dev.yml @@ -2,7 +2,7 @@ name: fastscapelib-python-dev channels: - conda-forge dependencies: - - python>=3.8 + - python>=3.10 - numpy - numba - xtensor-python diff --git a/include/fastscapelib/flow/flow_graph.hpp b/include/fastscapelib/flow/flow_graph.hpp index cc0e4d7d..87399da2 100644 --- a/include/fastscapelib/flow/flow_graph.hpp +++ b/include/fastscapelib/flow/flow_graph.hpp @@ -251,7 +251,7 @@ namespace fastscapelib * @tparam FK The flow kernel type. * @tparam FKD The flow kernel data type. * @param kernel The flow kernel object to apply along the graph. - * @param kernel_data The object holding or referencing input and output data + * @param data The object holding or referencing input and output data * used by the flow kernel. * */ diff --git a/include/fastscapelib/utils/thread_pool.hpp b/include/fastscapelib/utils/thread_pool.hpp index 1e42dffc..8a8f1a91 100644 --- a/include/fastscapelib/utils/thread_pool.hpp +++ b/include/fastscapelib/utils/thread_pool.hpp @@ -57,6 +57,7 @@ namespace fastscapelib * @param first_index_ The first index in the range. * @param index_after_last_ The index after the last index in the range. * @param num_blocks_ The desired number of blocks to divide the range into. + * @param min_size_ The minimum size of each blocks (ignored if zero, default). */ blocks(const T& first_index_, const T& index_after_last_, diff --git a/pyproject.toml b/pyproject.toml index 5ff3d5fd..5c30c672 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,11 @@ [build-system] requires = [ "scikit-build-core", - "pybind11", - # TODO: replace by "numpy>=1.25.0,<2" when dropping py3.8 support. - # https://github.com/scipy/oldest-supported-numpy/issues/76 - "oldest-supported-numpy", + # pybind11 2.12 added support for numpy 2.0 + # pybind11 doesn't require numpy at build time, but xtensor-python does! + # packages built with numpy 2.x are compatible with numpy 1.xx + "pybind11>=2.12,<3", + "numpy>=2.0,<3", ] build-backend = "scikit_build_core.build" @@ -21,14 +22,17 @@ authors = [ maintainers = [ {name = "Fastscapelib contributors"}, ] -requires-python = ">=3.8" -dependencies = ["numpy", "numba"] +requires-python = ">=3.10" +dependencies = [ + "numpy>=1.24", +] [project.urls] Home = "https://fastscapelib.readthedocs.io" Repository = "https://github.com/fastscape-lem/fastscapelib" [project.optional-dependencies] +numba = ["numba"] test = ["pytest>=6.0"] [tool.scikit-build] @@ -40,6 +44,12 @@ test-extras = "test" test-command = "pytest {project}/python/fastscapelib/tests" build-verbosity = 1 +[[tool.cibuildwheel.overrides]] +# TODO: remove when numba supports python 3.13 +select = "cp3{10,11,12}-*" +inherit.test-requires = "append" +test-requires = ["numba"] + [tool.cibuildwheel.macos] environment = {SKBUILD_CMAKE_ARGS='-DFS_DOWNLOAD_XTENSOR_PYTHON=ON', MACOSX_DEPLOYMENT_TARGET=10.13} diff --git a/python/fastscapelib/eroders/flow_kernel_eroder.py b/python/fastscapelib/eroders/flow_kernel_eroder.py index bdacdad7..340c945b 100644 --- a/python/fastscapelib/eroders/flow_kernel_eroder.py +++ b/python/fastscapelib/eroders/flow_kernel_eroder.py @@ -1,14 +1,19 @@ -import inspect +from __future__ import annotations -import numba as nb +import inspect +from typing import TYPE_CHECKING, Any from fastscapelib.flow import FlowGraph, FlowGraphTraversalDir -from fastscapelib.flow.numba_flow_kernel import ( - NumbaFlowKernel, - NumbaFlowKernelData, - NumbaJittedClass, - create_flow_kernel, -) +from fastscapelib.flow.numba import create_flow_kernel + +if TYPE_CHECKING: + import numba as nb + + from fastscapelib.flow.numba.flow_kernel import ( + NumbaFlowKernel, + NumbaFlowKernelData, + NumbaJittedClass, + ) class FlowKernelEroderMeta(type): @@ -46,7 +51,7 @@ def __new__(cls, name, bases, dct): class FlowKernelEroder(metaclass=FlowKernelEroderMeta): - spec: dict[str, nb.types.Type] + spec: dict[str, nb.types.Type | tuple[nb.types.Type, Any]] outputs: list[str] apply_dir: FlowGraphTraversalDir _flow_graph: FlowGraph diff --git a/python/fastscapelib/flow/__init__.py b/python/fastscapelib/flow/__init__.py index 5ecea173..de8068de 100644 --- a/python/fastscapelib/flow/__init__.py +++ b/python/fastscapelib/flow/__init__.py @@ -15,7 +15,7 @@ _FlowKernelData, ) -from fastscapelib.flow.numba_flow_kernel import create_flow_kernel +from fastscapelib.flow.numba import create_flow_kernel __all__ = [ "FlowDirection", diff --git a/python/fastscapelib/flow/__init__.pyi b/python/fastscapelib/flow/__init__.pyi index 280d6e03..9e2e3576 100644 --- a/python/fastscapelib/flow/__init__.pyi +++ b/python/fastscapelib/flow/__init__.pyi @@ -3,7 +3,7 @@ from typing import Any, ClassVar, List, Union, overload import numpy as np import numpy.typing as npt -from fastscapelib.flow.numba_flow_kernel import NumbaFlowKernel, NumbaFlowKernelData +from fastscapelib.flow.numba import NumbaFlowKernel, NumbaFlowKernelData from fastscapelib.grid import ProfileGrid, RasterGrid, TriMesh Grid = Union[ProfileGrid, RasterGrid, TriMesh] diff --git a/python/fastscapelib/flow/numba/__init__.py b/python/fastscapelib/flow/numba/__init__.py new file mode 100644 index 00000000..3c18c99f --- /dev/null +++ b/python/fastscapelib/flow/numba/__init__.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, Iterable + +if TYPE_CHECKING: + import numba as nb + + from fastscapelib.flow import FlowGraph, FlowGraphTraversalDir + from fastscapelib.flow.numba.flow_kernel import ( + NumbaFlowKernel, + NumbaFlowKernelData, + NumbaJittedClass, + ) + + +def create_flow_kernel( + flow_graph: FlowGraph, + kernel_func: Callable[[NumbaJittedClass], int], + spec: dict[str, nb.types.Type | tuple[nb.types.Type, Any]], + apply_dir: FlowGraphTraversalDir, + outputs: Iterable[str] = (), + max_receivers: int = -1, + n_threads: int = 1, + print_generated_code: bool = False, + print_stats: bool = False, +) -> tuple[NumbaFlowKernel, NumbaFlowKernelData]: + """Creates a numba flow kernel. + + Parameters + ---------- + flow_graph : :py:class:`~fastscapelib.FlowGraph` + A flow graph object. + kernel_func : callable + A Python function to apply to each node of the graph. It must take one + input argument (numba jit-compiled class) that holds or references input + and output kernel data of one graph node. It should return an integer. + spec : dict + Dictionary where keys are kernel input and output variable names and + values are either variable types (i.e. numba scalar or array types) or + variable (type, value) tuples (only for scalar variables). + apply_dir : :py:class:`~fastscapelib.FlowGraphTraversalDir.` + The direction and order in which the flow kernel will be applied along + the graph. + outputs : iterable + The names of the kernel output variables. All names given here must be + also present in ``spec``. + max_receivers : int + Maximum number of flow receiver nodes per graph node. Setting this number + to 1 may speed up the application of the kernel function along the graph. + Setting this number to -1 (default) will use the maximum number defined + from the flow graph object and is generally safer. + n_threads : int + Number of threads to use for applying the kernel function in parallel + along the flow graph (default: 1, the kernel will be applied sequentially). + A value > 1 may be useful for kernel functions that are computationally + expensive. For trivial kernel functions it is generally more efficient to + apply the kernel sequentially (i.e., ``n_threads = 1``). + print_generated_code : bool + If True, prints the code used to create the numba jit-compiled classes for + managing kernel data (default: False). Useful for debugging. + print_stats : bool + If True, prints a small report on kernel creation performance + (default: False). + + Returns + ------- + flow_kernel : :py:class:`~fastscapelib.flow.numba.flow_kernel.NumbaFlowKernel` + An object used to apply the flow kernel. + flow_kernel_data : :py:class:`~fastscapelib.flow.numba.flow_kernel.NumbaFlowKernelData` + An object used to manage flow kernel data. + + """ + from fastscapelib.flow.numba.flow_kernel import NumbaFlowKernelFactory + + factory = NumbaFlowKernelFactory( + flow_graph, + kernel_func, + spec, + apply_dir, + outputs=outputs, + max_receivers=max_receivers, + n_threads=n_threads, + print_generated_code=print_generated_code, + print_stats=print_stats, + ) + return factory.kernel, factory.data diff --git a/python/fastscapelib/flow/numba_flow_kernel.py b/python/fastscapelib/flow/numba/flow_kernel.py similarity index 88% rename from python/fastscapelib/flow/numba_flow_kernel.py rename to python/fastscapelib/flow/numba/flow_kernel.py index 401653a2..df1cba35 100644 --- a/python/fastscapelib/flow/numba_flow_kernel.py +++ b/python/fastscapelib/flow/numba/flow_kernel.py @@ -2,7 +2,6 @@ import ast import inspect -import sys import time from collections import defaultdict from collections.abc import Mapping @@ -10,12 +9,12 @@ from dataclasses import dataclass from textwrap import dedent, indent from typing import ( - TYPE_CHECKING, Any, Callable, Iterable, Iterator, Optional, + ParamSpec, Protocol, Type, TypeVar, @@ -26,27 +25,12 @@ import numpy as np from numba.experimental.jitclass import _box # type: ignore[missing-imports] -# type stubs defined in this sub-package (avoid circular imports) -if TYPE_CHECKING: - from fastscapelib.flow import ( - FlowGraph, - FlowGraphTraversalDir, - _FlowKernel, - _FlowKernelData, - ) -else: - from _fastscapelib_py.flow import ( - FlowGraph, - FlowGraphTraversalDir, - _FlowKernel, - _FlowKernelData, - ) - - -if sys.version_info < (3, 10): - from typing_extensions import ParamSpec -else: - from typing import ParamSpec +from fastscapelib.flow import ( + FlowGraph, + FlowGraphTraversalDir, + _FlowKernel, + _FlowKernelData, +) class ConstantAssignmentVisitor(ast.NodeVisitor): @@ -103,24 +87,10 @@ class NumbaJittedClass(Protocol): _numba_type_: Any -if sys.version_info < (3, 10): - KernelFunc = "NumbaJittedFunc[[NumbaJittedClass], int]" - KernelNodeDataGetter = ( - "NumbaJittedFunc[[int, NumbaJittedClass, NumbaJittedClass], int]" - ) - KernelNodeDataSetter = ( - "NumbaJittedFunc[[int, NumbaJittedClass, NumbaJittedClass], int]" - ) - KernelNodeDataCreate = "NumbaJittedFunc[[], NumbaJittedClass]" -else: - KernelFunc = NumbaJittedFunc[[NumbaJittedClass], int] - KernelNodeDataGetter = NumbaJittedFunc[ - [int, NumbaJittedClass, NumbaJittedClass], int - ] - KernelNodeDataSetter = NumbaJittedFunc[ - [int, NumbaJittedClass, NumbaJittedClass], int - ] - KernelNodeDataCreate = NumbaJittedFunc[[], NumbaJittedClass] +KernelFunc = NumbaJittedFunc[[NumbaJittedClass], int] +KernelNodeDataGetter = NumbaJittedFunc[[int, NumbaJittedClass, NumbaJittedClass], int] +KernelNodeDataSetter = NumbaJittedFunc[[int, NumbaJittedClass, NumbaJittedClass], int] +KernelNodeDataCreate = NumbaJittedFunc[[], NumbaJittedClass] @contextmanager @@ -254,77 +224,6 @@ class NumbaFlowKernel: func: KernelFunc -def create_flow_kernel( - flow_graph: FlowGraph, - kernel_func: Callable[["NumbaJittedClass"], int], - spec: dict[str, nb.types.Type | tuple[nb.types.Type, Any]], - apply_dir: FlowGraphTraversalDir, - outputs: Iterable[str] = (), - max_receivers: int = -1, - n_threads: int = 1, - print_generated_code: bool = False, - print_stats: bool = False, -) -> tuple[NumbaFlowKernel, NumbaFlowKernelData]: - """Creates a numba flow kernel. - - Parameters - ---------- - flow_graph : :py:class:`~fastscapelib.FlowGraph` - A flow graph object. - kernel_func : callable - A Python function to apply to each node of the graph. It must take one - input argument (numba jit-compiled class) that holds or references input - and output kernel data of one graph node. It should return an integer. - spec : dict - Dictionary where keys are kernel input and output variable names and - values are either variable types (i.e. numba scalar or array types) or - variable (type, value) tuples (only for scalar variables). - apply_dir : :py:class:`~fastscapelib.FlowGraphTraversalDir.` - The direction and order in which the flow kernel will be applied along - the graph. - outputs : iterable - The names of the kernel output variables. All names given here must be - also present in ``spec``. - max_receivers : int - Maximum number of flow receiver nodes per graph node. Setting this number - to 1 may speed up the application of the kernel function along the graph. - Setting this number to -1 (default) will use the maximum number defined - from the flow graph object and is generally safer. - n_threads : int - Number of threads to use for applying the kernel function in parallel - along the flow graph (default: 1, the kernel will be applied sequentially). - A value > 1 may be useful for kernel functions that are computationally - expensive. For trivial kernel functions it is generally more efficient to - apply the kernel sequentially (i.e., ``n_threads = 1``). - print_generated_code : bool - If True, prints the code used to create the numba jit-compiled classes for - managing kernel data (default: False). Useful for debugging. - print_stats : bool - If True, prints a small report on kernel creation performance - (default: False). - - Returns - ------- - flow_kernel : :py:class:`~fastscapelib.flow.numba_flow_kernel.NumbaFlowKernel` - An object used to apply the flow kernel. - flow_kernel_data : :py:class:`~fastscapelib.flow.numba_flow_kernel.NumbaFlowKernelData` - An object used to manage flow kernel data. - - """ - factory = NumbaFlowKernelFactory( - flow_graph, - kernel_func, - spec, - apply_dir, - outputs=outputs, - max_receivers=max_receivers, - n_threads=n_threads, - print_generated_code=print_generated_code, - print_stats=print_stats, - ) - return factory.kernel, factory.data - - class NumbaFlowKernelFactory: """Creates a numba flow kernel. diff --git a/python/fastscapelib/tests/test_flow_kernel.py b/python/fastscapelib/tests/test_flow_kernel.py index 4363d177..65238bff 100644 --- a/python/fastscapelib/tests/test_flow_kernel.py +++ b/python/fastscapelib/tests/test_flow_kernel.py @@ -1,4 +1,3 @@ -import numba as nb import numpy as np import pytest @@ -8,9 +7,11 @@ MultiFlowRouter, PFloodSinkResolver, ) -from fastscapelib.flow.numba_flow_kernel import create_flow_kernel +from fastscapelib.flow.numba import create_flow_kernel from fastscapelib.grid import NodeStatus, RasterGrid +nb = pytest.importorskip("numba") + @pytest.fixture(scope="module") def flow_graph(): diff --git a/python/src/flow_graph.cpp b/python/src/flow_graph.cpp index 14c120ca..30e5f3e1 100644 --- a/python/src/flow_graph.cpp +++ b/python/src/flow_graph.cpp @@ -626,7 +626,8 @@ add_flow_graph_bindings(py::module& m) { auto py_apply_kernel = py::module::import("fastscapelib") .attr("flow") - .attr("numba_flow_kernel") + .attr("numba") + .attr("flow_kernel") .attr("apply_flow_kernel"); return py_apply_kernel(flow_graph, flow_kernel, flow_kernel_data).cast(); } @@ -650,9 +651,9 @@ add_flow_graph_bindings(py::module& m) Parameters ---------- - kernel : :py:class:`~fastscapelib.flow.numba_flow_kernel.NumbaFlowKernel` + kernel : :py:class:`~fastscapelib.flow.numba.flow_kernel.NumbaFlowKernel` The flow kernel object to apply along the graph. - kernel_data : :py:class:`~fastscapelib.flow.numba_flow_kernel.NumbaFlowKernelData` + kernel_data : :py:class:`~fastscapelib.flow.numba.flow_kernel.NumbaFlowKernelData` The object holding or referencing input and output data used by the flow kernel. )doc");