Merge pull request #241 from ska-sa/arm64-wheels-v2

Build aarch64 wheels

.github/workflows/test.yml

@@ -194,6 +194,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
+      - uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
       - uses: pypa/cibuildwheel@v2.15.0
       - uses: actions/upload-artifact@v3
         with:

configure.ac

@@ -38,18 +38,25 @@ CXXFLAGS="$save_CXXFLAGS"
 
 ### Required libraries
 
-SPEAD2_CHECK_LIB([boost/program_options.hpp], [boost_program_options],
-                 [boost::program_options::option_description foo],
-                 [], [AC_MSG_ERROR([boost_program_options is required])])
-SPEAD2_CHECK_LIB([boost/system/system_error.hpp], [boost_system], [boost::system::error_code ec],
-                 [], [AC_MSG_ERROR([boost_system is required])])
-SPEAD2_CHECK_HEADER([boost/asio.hpp], [boost_system,pthread], [boost::asio::io_service io_service],
+SPEAD2_CHECK_HEADER([boost/asio.hpp], [pthread], [boost::asio::io_service io_service],
                     [], [AC_MSG_ERROR([boost_asio is required])])
 SPEAD2_CHECK_HEADER([libdivide.h], [], [libdivide::divider<unsigned long long> div],
                     [], [AC_MSG_ERROR([libdivide is required])])
 
 ### Optional libraries/features
 
+SPEAD2_ARG_WITH(
+    [program_options],
+    [AS_HELP_STRING([--without-program-options], [Do not build utility programs])],
+    [SPEAD2_USE_PROGRAM_OPTIONS],
+    [SPEAD2_CHECK_LIB(
+        [boost/program_options.hpp], [boost_program_options],
+        [boost::program_options::option_description foo],
+        [SPEAD2_USE_PROGRAM_OPTIONS=1],
+        [AC_MSG_ERROR([boost_program_options is required unless --without-program-options is given])]
+    )]
+)
+
 # pkg-config files for libibverbs are quite new, so fall back to searching
 # system paths.
 IBV_CFLAGS=""
@@ -219,7 +226,7 @@ SPEAD2_ARG_WITH(
 
 ### Determine libraries to link against and include paths
 
-LIBS="-lboost_system -lpthread -ldl"
+LIBS="-lpthread -ldl"
 SPEAD2_CFLAGS=""
 AS_IF([test "x$SPEAD2_USE_PCAP" = "x1"], [
     LIBS="$PCAP_LIBS $LIBS"
@@ -266,6 +273,7 @@ AM_CONDITIONAL([SPEAD2_USE_IBV], [test "x$SPEAD2_USE_IBV" = "x1"])
 AM_CONDITIONAL([SPEAD2_USE_CUDA], [test "x$SPEAD2_USE_CUDA" = "x1"])
 AM_CONDITIONAL([SPEAD2_USE_GDRAPI], [test "x$SPEAD2_USE_GDRAPI" = "x1"])
 AM_CONDITIONAL([SPEAD2_USE_CAP], [test "x$SPEAD2_USE_CAP" = "x1"])
+AM_CONDITIONAL([SPEAD2_USE_PROGRAM_OPTIONS], [test "x$SPEAD2_USE_PROGRAM_OPTIONS" = "x1"])
 
 AC_SUBST(SPEAD2_MAJOR, m4_esyscmd([python3 gen/get_version.py major]))
 AC_SUBST(SPEAD2_MINOR, m4_esyscmd([python3 gen/get_version.py minor]))

doc/introduction.rst

@@ -72,19 +72,20 @@ Python install from source
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 Installing from source requires a modern C++ compiler supporting C++11 (GCC
 4.8+ or Clang 3.5+, although only GCC 11.2 and Clang 14.0 are tested and support
-for older compilers may be dropped) as well as Boost (including compiled
-libraries), libdivide, and the Python development headers. At the moment only
-GNU/Linux and OS X get tested but other POSIX-like systems should work too.
-There are no plans to support Windows.
+for older compilers may be dropped) as well as Boost 1.69+
+(only headers are required), libdivide, and the Python development headers.
+At the moment only GNU/Linux and OS X get tested but other POSIX-like systems
+should work too.  There are no plans to support Windows.
 
 Installation works with standard Python installation methods.
 
 Installing spead2 for C++
 -------------------------
 spead2 requires a modern C++ compiler supporting C++11 (see above for supported
-compilers) as well as Boost (including compiled libraries) and libdivide. At
-the moment only GNU/Linux and OS X get tested but other POSIX-like systems
-should work too.  There are no plans to support Windows.
+compilers) as well as Boost 1.69+ (including the compiled boost_program_options
+library) and libdivide. At the moment only GNU/Linux and OS X get tested but
+other POSIX-like systems should work too. There are no plans to support
+Windows.
 
 The C++ API uses the standard autoconf installation flow i.e.:
 

manylinux/before_all.sh

@@ -7,9 +7,11 @@ set -e -u
 package="$1"
 
 yum install -y \
-    wget libpcap libpcap-devel python-devel \
-    cmake3 ninja-build pandoc libnl3-devel \
-    ccache
+    wget libpcap libpcap-devel \
+    cmake3 ninja-build libnl3-devel
+if [[ "${CC:-}" == ccache* ]]; then
+    yum install -y ccache
+fi
 
 # Workaround for https://github.com/pypa/manylinux/issues/1203
 unset SSL_CERT_FILE
@@ -21,9 +23,8 @@ unset LDFLAGS
 # Install boost
 wget --progress=dot:mega https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2 -O /tmp/boost_1_81_0.tar.bz2
 tar -C /tmp -jxf /tmp/boost_1_81_0.tar.bz2
-cd /tmp/boost_1_81_0
-./bootstrap.sh --prefix=/usr --with-libraries=program_options,system
-./b2 cxxflags=-fPIC link=static install
+# Quick-n-dirty approach (much faster than doing the install, which copies thousands of files)
+ln -s /tmp/boost_1_81_0/boost /usr/include/boost
 
 # Install rdma-core
 wget --progress=dot:mega https://github.com/linux-rdma/rdma-core/releases/download/v44.0/rdma-core-44.0.tar.gz -O /tmp/rdma-core-44.0.tar.gz

pyproject.toml

@@ -7,16 +7,24 @@ before-all = "manylinux/before_all.sh {package}"
 before-build = "manylinux/before_build.sh"
 manylinux-x86_64-image="manylinux2014"
 manylinux-i686-image="manylinux2014"
-manylinux-aarch64-image="manylinux2014"
+# The manylinux2014 AArch64 images don't provide cmake, so build 2_28 wheels instead
+manylinux-aarch64-image="manylinux_2_28"
 build = ["cp38-manylinux*", "cp39-manylinux*", "cp310-manylinux*", "cp311-manylinux*", "cp312-manylinux*"]
 
 before-test = "pip install -r {package}/requirements.txt -r {package}/requirements-3.12.txt"
 test-command = "pytest -v {package}/tests"
 
 [tool.cibuildwheel.linux]
-archs = ["x86_64"]
+archs = ["aarch64", "x86_64"]
 
 [tool.cibuildwheel.linux.environment]
+CFLAGS = "-flto"
+LDFLAGS = "-flto"
+
+# ccache isn't available on all architectures, so only use it conditionally
+[[tool.cibuildwheel.overrides]]
+select = "*linux_x86_64"
+[tool.cibuildwheel.overrides.environment]
 CC = "ccache gcc"
 CFLAGS = "-flto"
 LDFLAGS = "-flto"

setup.py

@@ -43,7 +43,9 @@ def initialize_options(self):
 
     def run(self):
         self.mkpath(self.build_temp)
-        subprocess.check_call(os.path.abspath("configure"), cwd=self.build_temp)
+        subprocess.check_call(
+            [os.path.abspath("configure"), "--without-program-options"], cwd=self.build_temp
+        )
         config = configparser.ConfigParser()
         config.read(os.path.join(self.build_temp, "python-build.cfg"))
         for extension in self.extensions:

src/Makefile.am

@@ -20,6 +20,8 @@ bin_PROGRAMS = spead2_recv spead2_send spead2_bench
 check_PROGRAMS = spead2_unittest
 TESTS = spead2_unittest
 
+if SPEAD2_USE_PROGRAM_OPTIONS
+
 if SPEAD2_USE_IBV
 bin_PROGRAMS += mcdump
 mcdump_SOURCES = mcdump.cpp
@@ -35,6 +37,8 @@ spead2_send_LDADD = -lboost_program_options $(LDADD)
 spead2_bench_SOURCES = spead2_bench.cpp spead2_cmdline.cpp
 spead2_bench_LDADD = -lboost_program_options $(LDADD)
 
+endif
+
 if SPEAD2_USE_CAP
 bin_PROGRAMS += spead2_net_raw
 spead2_net_raw_SOURCES = spead2_net_raw.cpp

tests/test_passthrough.py

@@ -15,6 +15,7 @@
 
 """Test that data can be passed over the SPEAD protocol using the various transports."""
 
+import ipaddress
 import os
 import socket
 import sys
@@ -62,18 +63,30 @@ class BaseTestPassthrough:
 
     is_lossy = False
     requires_ipv6 = False
+    requires_ipv4_multicast = False
     requires_ipv6_multicast = False
 
     @classmethod
-    def check_ipv6(cls):
-        if not socket.has_ipv6:
-            pytest.skip("platform does not support IPv6")
-        with socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) as sock:
-            # Travis build systems fail to bind to an IPv6 address
-            try:
-                sock.bind(("::1", 8888))
-            except OSError:
-                pytest.skip("platform cannot bind IPv6 localhost address")
+    def check_platform(cls):
+        if cls.requires_ipv6:
+            if not socket.has_ipv6:
+                pytest.skip("platform does not support IPv6")
+            with socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) as sock:
+                # Travis build systems fail to bind to an IPv6 address
+                try:
+                    sock.bind(("::1", 8888))
+                except OSError:
+                    pytest.skip("platform cannot bind IPv6 localhost address")
+        if cls.requires_ipv4_multicast:
+            with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+                # qemu doesn't yet support IP_MULTICAST_IF socket option
+                # (https://gitlab.com/qemu-project/qemu/-/issues/1837)
+                # Skip the test if it is non-functional.
+                try:
+                    loopback = ipaddress.ip_address("127.0.0.1")
+                    sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, loopback.packed)
+                except OSError:
+                    pytest.skip("platform cannot set multicast interface (might be qemu?)")
         if cls.requires_ipv6_multicast:
             with socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) as sock:
                 # Github Actions on MacOS doesn't have routes to multicast
@@ -290,8 +303,7 @@ def transmit_item_groups(
         be transmitted over a separate substream (thus, the transport must
         support substreams if `item_groups` has more than one element).
         """
-        if self.requires_ipv6:
-            self.check_ipv6()
+        self.check_platform()
         recv_config = spead2.recv.StreamConfig(memcpy=memcpy)
         if allocator is not None:
             recv_config.memory_allocator = allocator
@@ -528,6 +540,7 @@ def prepare_senders(self, thread_pool, n):
 
 class TestPassthroughUdpMulticast(BaseTestPassthroughSubstreams):
     is_lossy = True
+    requires_ipv4_multicast = True
     MCAST_GROUP = "239.255.88.88"
     INTERFACE_ADDRESS = "127.0.0.1"
 

tests/test_passthrough_asyncio.py

@@ -49,8 +49,7 @@ def transmit_item_groups(
     async def transmit_item_groups_async(
         self, item_groups, *, memcpy, allocator, new_order="=", group_mode=None
     ):
-        if self.requires_ipv6:
-            self.check_ipv6()
+        self.check_platform()
         recv_config = spead2.recv.StreamConfig(memcpy=memcpy)
         if allocator is not None:
             recv_config.memory_allocator = allocator