feat: Python wheels workflow and build backend

.gitignore

@@ -4,13 +4,17 @@ branches/
 build/
 dist/
 ext/
+wheelhouse/
+_coverage/
+.python-version
 .idea
 .vscode
 .cproject
 .project
 .DS_Store
 *.pyc
-_coverage/
+/*.lock
+gastest.o
 
 # Exclude test files I tend to leave around at the top level. The leading
 # slash ensures that files with these extensions within subdirectories are not
@@ -21,3 +25,5 @@ _coverage/
 /*.jxl
 /*.tx
 /*.log
+
+

CMakeLists.txt

@@ -248,10 +248,16 @@ if (NOT EMBEDPLUGINS AND NOT BUILD_OIIOUTIL_ONLY)
     endforeach ()
 endif ()
 
-if (USE_PYTHON AND Python3_Development_FOUND AND NOT BUILD_OIIOUTIL_ONLY)
+
+if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0" AND NOT WIN32)
+    set (_py_dev_found Python3_Development.Module_FOUND)
+else ()
+    set (_py_dev_found Python3_Development_FOUND)
+endif ()
+if (USE_PYTHON AND ${_py_dev_found} AND NOT BUILD_OIIOUTIL_ONLY)
     add_subdirectory (src/python)
 else ()
-    message (STATUS "Not building Python bindings: USE_PYTHON=${USE_PYTHON}, Python3_Development_FOUND=${Python3_Development_FOUND}")
+    message (STATUS "Not building Python bindings: USE_PYTHON=${USE_PYTHON}, Python3_Development.Module_FOUND=${Python3_Development.Module_FOUND}")
 endif ()
 
 add_subdirectory (src/include)

INSTALL.md

@@ -357,6 +357,28 @@ produce a dynamic-linked version.
 
 3. Execute the PowerShell command from where vcpkg is located in directory. ``vcpkg install openimageio``
 
+
+Python-based Builds and Installs
+--------------------------------
+
+**Installing from prebuilt binary distributions**
+
+If you're only interested in the Python module and the CLI tools, you can install with `pip` or `uv`:
+
+> ```pip install OpenImageIO```
+
+**Building and installing from source**
+
+If you have a C++ compiler installed, you can also use the Python build-backend to compile and install
+from source by navigating to the root of the repository and running: ```pip install .```
+
+This will download and install CMake and Ninja if necessary, and invoke the CMake build system; which,
+in turn, will build missing dependencies, compile OIIO, and install the Python module, the libraries, 
+the headers, and the CLI tools to a platform-specific, Python-specific location. 
+
+See the [scikit-build-core docs](https://scikit-build-core.readthedocs.io/en/latest/configuration.html#configuring-cmake-arguments-and-defines)
+for more information on customizing and overriding build-tool options and CMake arguments.
+
 Test Images
 -----------
 

pyproject.toml

@@ -0,0 +1,170 @@
+[project]
+name = "OpenImageIO"
+# The build backend ascertains the version from the CMakeLists.txt file.
+dynamic = ["version"]
+description = "Reading, writing, and processing images in a wide variety of file formats, using a format-agnostic API, aimed at VFX applications."
+authors = [
+    {name = "Larry Gritz", email = "lg@larrygritz.com"},
+    {name = "OpenImageIO Contributors", email = "oiio-dev@lists.aswf.io"}
+]
+maintainers = [
+    {name = "OpenImageIO Contributors", email="oiio-dev@lists.aswf.io"},
+]
+readme = "README.md"
+license = {text = "Apache-2.0"}
+keywords = [""]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: Apache Software License",
+    "Topic :: Multimedia :: Graphics",
+    "Topic :: Multimedia :: Video",
+    "Topic :: Multimedia :: Video :: Display",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = ["numpy"]
+requires-python = ">= 3.8"
+
+[project.urls]
+Homepage = "https://openimageio.org/"
+Source = "https://github.com/AcademySoftwareFoundation/OpenImageIO"
+Documentation = "https://openimageio.readthedocs.io"
+Issues = "https://github.com/AcademySoftwareFoundation/OpenImageIO/issues"
+
+
+[project.scripts]
+# Use the convention below to expose CLI tools as Python scripts.
+iconvert = "OpenImageIO:_command_line"
+idiff = "OpenImageIO:_command_line"
+igrep = "OpenImageIO:_command_line"
+iinfo = "OpenImageIO:_command_line"
+maketx = "OpenImageIO:_command_line"
+oiiotool = "OpenImageIO:_command_line"
+testtex = "OpenImageIO:_command_line"
+
+[build-system]
+build-backend = "scikit_build_core.build"
+requires = [
+    "scikit-build-core>=0.10.6,<1",
+    "pybind11",
+    "numpy>=2.0,<3"
+]
+
+[tool.scikit-build]
+build.verbose = true
+# Exclude unnecessary directories from the source distribution.
+sdist.exclude = [".github", "testsuite", "ASWF", "docs"]
+# Pin the minimum scikit-build-core and CMake versions to those specified
+# in build-system.requires and CMakeLists.txt, respectively.
+minimum-version = "build-system.requires"
+cmake.version = "CMakeLists.txt"
+wheel.license-files = ["LICENSE.md", "THIRD-PARTY.md"]
+# Make sure the package is structured as expected.
+wheel.install-dir = "OpenImageIO"
+
+
+[tool.scikit-build.cmake.define]
+# Build missing dependencies. See src/cmake for details.
+OpenImageIO_BUILD_MISSING_DEPS = "all"
+# Don't build tests. Dramatically improves build time.
+OIIO_BUILD_TESTS = "0"
+# Prefer linking static dependencies when possible.
+LINKSTATIC = "1"
+
+
+# Dynamically set the package version metadata by pasrsing CMakeLists.txt.
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "CMakeLists.txt" 
+regex = 'set \(OpenImageIO_VERSION "(?P<value>[0-9a-z.]+)"\)'
+
+# On macOS, make sure dependencies are built for the correct architecture. 
+[[tool.scikit-build.overrides]]
+if.platform-system = "darwin"
+if.platform-machine  = "arm64"
+inherit.cmake.define = "append"
+cmake.define.CMAKE_OSX_ARCHITECTURES = "arm64"
+
+[[tool.scikit-build.overrides]]
+if.platform-system = "darwin"
+if.platform-machine  = "x86_64"
+inherit.cmake.define = "append"
+cmake.define.CMAKE_OSX_ARCHITECTURES = "x86_64"
+
+[[tool.scikit-build.overrides]]
+if.platform-system = "linux"
+inherit.cmake.define = "append"
+cmake.define.CMAKE_CXX_FLAGS = "-Wno-error=stringop-overflow -Wno-pragmas"
+
+[tool.cibuildwheel]
+build-verbosity = 1
+skip = [
+    # Skip 32-bit builds
+    "*-win32",
+    "*-manylinux_i686",
+    # Building with musl seems to work, but the repair-wheel step seems to fail...
+    # This may be a bug in repairwheel (or auditwheel)?
+    "*musllinux*",  
+]
+# If the following command returns code 0, we know the CLI Python scripts
+# are able to import the OpenImageIO module, which is able to invoke the
+# CLI binary executables and find and load all shared libraries. The 
+# '--help' flag outputs a list of dependencies with which OIIO was compiled.
+test-command = "oiiotool --help"
+# Enable free-threaded support
+#free-threaded-support = true
+# Default images to use when building manylinux wheels locally.
+manylinux-x86_64-image = "manylinux_2_28"
+manylinux-aarch64-image = "manylinux_2_28"
+manylinux-pypy_x86_64-image = "manylinux_2_28"
+manylinux-pypy_aarch64-image = "manylinux_2_28"
+musllinux-x86_64-image = "musllinux_1_2"
+musllinux-aarch64-image = "musllinux_1_2"
+
+[tool.cibuildwheel.macos.environment]
+# Specify a build directory so the repairwheel command can (re)find
+# dependency libraries built by OpenImageIO.
+SKBUILD_BUILD_DIR = "/tmp/build_oiio_wheels"
+# Always (re)build the TIFF library; otherwise, OIIO has trouble
+# with libdeflate during successive incremental builds.
+SKBUILD_CMAKE_ARGS = "-DOpenImageIO_BUILD_LOCAL_DEPS=TIFF; -DOpenImageIO_BUILD_MISSING_DEPS=all"
+# std::filesystem is only available in macOS 10.15 and later.
+MACOSX_DEPLOYMENT_TARGET = "10.15"
+# Optimize for size (not speed)
+SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
+
+[tool.cibuildwheel.linux.environment]
+SKBUILD_BUILD_DIR = "/tmp/build_oiio_wheels"
+SKBUILD_CMAKE_ARGS = "-DOpenImageIO_BUILD_LOCAL_DEPS=all; -DOpenImageIO_BUILD_MISSING_DEPS=all"
+# Suppress warnings that cause linux cibuildwheel build to fail
+CXXFLAGS = "-Wno-error=stringop-overflow -Wno-pragmas"
+SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
+
+[tool.cibuildwheel.windows.environment]
+SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
+
+
+# On macOS and Linux, run a custom repair-wheel-command after the build.
+# See tasks.py for more details.
+[tool.cibuildwheel.macos]
+before-build = "pip install 'repairwheel' && pip install invoke && brew uninstall --ignore-dependencies harfbuzz fontconfig cairo freetype"
+repair-wheel-command = "invoke wheel-repair --build-dir ${SKBUILD_BUILD_DIR} --wheel-path {wheel} --output-dir {dest_dir}"
+
+[tool.cibuildwheel.linux]
+before-build = "pip install repairwheel && pip install invoke"
+repair-wheel-command = "invoke wheel-repair --build-dir ${SKBUILD_BUILD_DIR} --wheel-path {wheel} --output-dir {dest_dir}"
+
+
+[[tool.cibuildwheel.overrides]]
+select = "{c,p}p3{7,8}-*"
+manylinux-x86_64-image = "manylinux2014"
+manylinux-aarch64-image = "manylinux2014"
+manylinux-pypy_x86_64-image = "manylinux2014"

src/build-scripts/build_libtiff.bash

@@ -45,6 +45,7 @@ if [[ -z $DEP_DOWNLOAD_ONLY ]]; then
                -Dtiff-tests=${LIBTIFF_BUILD_TESTS:-OFF} \
                -Dtiff-docs=${LIBTIFF_BUILD_TESTS:-OFF} \
                -Dlibdeflate=ON \
+               -Dwebp=OFF \ 
                ${LIBTIFF_BUILDOPTS}
     time cmake --build build --target install
 fi

src/cmake/build_libdeflate.cmake

@@ -9,7 +9,7 @@
 set_cache (libdeflate_BUILD_VERSION 1.20 "libdeflate version for local builds")
 set (libdeflate_GIT_REPOSITORY "https://github.com/ebiggers/libdeflate")
 set (libdeflate_GIT_TAG "v${libdeflate_BUILD_VERSION}")
-set_cache (libdeflate_BUILD_SHARED_LIBS  ${LOCAL_BUILD_SHARED_LIBS_DEFAULT}
+set_cache (libdeflate_BUILD_SHARED_LIBS OFF # ${LOCAL_BUILD_SHARED_LIBS_DEFAULT}
            DOC "Should a local libdeflate build, if necessary, build shared libraries" ADVANCED)
 
 string (MAKE_C_IDENTIFIER ${libdeflate_BUILD_VERSION} libdeflate_VERSION_IDENT)
@@ -20,6 +20,7 @@ build_dependency_with_cmake(libdeflate
     GIT_TAG         ${libdeflate_GIT_TAG}
     CMAKE_ARGS
         -D BUILD_SHARED_LIBS=${libdeflate_BUILD_SHARED_LIBS}
+        -D LIBDEFLATE_BUILD_SHARED_LIB=${libdeflate_BUILD_SHARED_LIBS}
         -D CMAKE_POSITION_INDEPENDENT_CODE=ON
         -D CMAKE_INSTALL_LIBDIR=lib
         -D LIBDEFLATE_BUILD_GZIP=OFF
@@ -32,6 +33,7 @@ set (libdeflate_ROOT ${libdeflate_LOCAL_INSTALL_DIR})
 # Signal to caller that we need to find again at the installed location
 set (libdeflate_REFIND TRUE)
 set (libdeflate_REFIND_ARGS CONFIG)
+set (libdeflate_REFIND_VERSION ${libdeflate_BUILD_VERSION})
 
 if (libdeflate_BUILD_SHARED_LIBS)
     install_local_dependency_libs (libdeflate libdeflate)

src/cmake/fancy_add_executable.cmake

@@ -53,6 +53,17 @@ macro (fancy_add_executable)
         target_link_libraries (${_target_NAME} PRIVATE ${_target_LINK_LIBRARIES})
         target_link_libraries (${_target_NAME} PRIVATE ${PROFILER_LIBRARIES})
         set_target_properties (${_target_NAME} PROPERTIES FOLDER ${_target_FOLDER})
+        if (SKBUILD) 
+            # When building cli tools with scikit-build, set RPATH
+            # to a relative path to the distribution's lib directory.
+            if (APPLE)
+                set_target_properties (${_target_NAME} PROPERTIES 
+                        INSTALL_RPATH "@loader_path/../${CMAKE_INSTALL_LIBDIR}")
+            elseif (LINUX)
+                set_target_properties (${_target_NAME} PROPERTIES 
+                        INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")
+            endif ()
+        endif ()
         check_is_enabled (INSTALL_${_target_NAME} _target_NAME_INSTALL_enabled)
         if (CMAKE_UNITY_BUILD AND UNITY_BUILD_MODE STREQUAL GROUP)
             set_source_files_properties(${_target_SRC} PROPERTIES

src/cmake/pythonutils.cmake

@@ -32,13 +32,24 @@ macro (find_python)
             list (APPEND _req EXACT)
         endif ()
     endif ()
+
+    # Support building on manylinux docker images, which do not contain
+    # the Development.Embedded component.
+    # https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode
+    if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0" AND NOT WIN32)
+        set (_py_components Interpreter Development.Module)
+    else ()
+        set (_py_components Interpreter Development)
+    endif ()
+
     checked_find_package (Python3 ${PYTHON_VERSION}
                           ${_req}
                           VERSION_MIN 3.7
-                          COMPONENTS Interpreter Development
+                          COMPONENTS ${_py_components}
                           PRINT Python3_VERSION Python3_EXECUTABLE
                                 Python3_LIBRARIES
                                 Python3_Development_FOUND
+                                Python3_Development.Module_FOUND
                                 Python3_Interpreter_FOUND )
 
     # The version that was found may not be the default or user
@@ -129,6 +140,19 @@ macro (setup_python_module)
 #                               SUFFIX ".pyd")
 #    endif()
 
+    if (SKBUILD)
+        set (PYTHON_SITE_DIR .)
+        # When building python wheels, set RPATH to a relative path to 
+        # the distribution's lib directory.
+        if (APPLE)
+            set_target_properties (${target_name} PROPERTIES 
+                    INSTALL_RPATH "@loader_path/${CMAKE_INSTALL_LIBDIR}")
+        elseif (LINUX)
+            set_target_properties (${target_name} PROPERTIES
+                    INSTALL_RPATH "$ORIGIN/${CMAKE_INSTALL_LIBDIR}")
+        endif ()
+    endif ()
+
     # In the build area, put it in lib/python so it doesn't clash with the
     # non-python libraries of the same name (which aren't prefixed by "lib"
     # on Windows).