Merge pull request #2775 from div72/arm64-darwin-cross

depends: add cross arm64-darwin support

.github/workflows/cd.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        script-id: [win32, win64, mac]
+        script-id: [win32, win64, mac, mac_arm64_cross]
     env:
       FILE_ENV: ./cd/00_setup_env_${{ matrix.script-id }}.sh
       OS_NAME: linux

.github/workflows/ci.yml

@@ -28,8 +28,10 @@ jobs:
                   # FIXME: depends is unable to compile Qt with clang.
                   # - name: x86_64 Linux  [TSan]  [GUI]  [jammy]
                   #  script-id: native_tsan
-                  - name: macOS 10.14  [GOAL deploy]  [GUI]  [no tests]  [focal]
+                  - name: macOS 10.14  [GOAL deploy]  [GUI]  [no tests]  [noble]
                     script-id: mac
+                  - name: arm64 macOS 10.14  [GOAL deploy]  [GUI]  [no tests]  [noble]
+                    script-id: mac_arm64_cross
         env:
             FILE_ENV: ./ci/test/00_setup_env_${{ matrix.script-id }}.sh
             OS_NAME: linux

Makefile.am

@@ -150,7 +150,7 @@ $(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE_DPIF
 	$(TIFFCP) -c none $(OSX_BACKGROUND_IMAGE_DPIFILES) $@
 
 $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/gridcoinresearch: $(OSX_APP_BUILT) $(OSX_PACKAGING)
-	INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -verbose 2
+	OBJDUMP=$(OBJDUMP) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -verbose 2
 
 deploydir: $(APP_DIST_EXTRAS)
 endif

cd/00_setup_env_mac.sh

@@ -7,9 +7,9 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos_cross
-export DOCKER_NAME_TAG=ubuntu:20.04
+export DOCKER_NAME_TAG=ubuntu:24.04
 export HOST=x86_64-apple-darwin
-export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 python3-dev python3-setuptools xorriso"
-export XCODE_VERSION=12.1
-export XCODE_BUILD_ID=12A7403
+export PACKAGES="clang lld llvm cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso"
+export XCODE_VERSION=15.0
+export XCODE_BUILD_ID=15A240d
 export GRIDCOIN_CONFIG="--with-gui --enable-reduce-exports"

cd/00_setup_env_mac_arm64_cross.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=cd_macos_arm64_cross
+export DOCKER_NAME_TAG=ubuntu:24.04
+export HOST=arm64-apple-darwin
+export PACKAGES="clang lld llvm cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso"
+export XCODE_VERSION=15.0
+export XCODE_BUILD_ID=15A240d
+export GRIDCOIN_CONFIG="--with-gui --enable-reduce-exports"

cd/00_setup_env_win32.sh

@@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_win32
 export DOCKER_NAME_TAG=ubuntu:20.04
 export HOST=i686-w64-mingw32
-export PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt wine32"
+export PACKAGES="cmake python3 nsis g++-mingw-w64-i686 wine-binfmt wine32"
 export GOAL="deploy"
 export GRIDCOIN_CONFIG="--enable-reduce-exports --with-gui=qt5"
 export DPKG_ADD_ARCH="i386"

cd/00_setup_env_win64.sh

@@ -9,5 +9,5 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_win64
 export DOCKER_NAME_TAG=ubuntu:20.04
 export HOST=x86_64-w64-mingw32
-export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
+export PACKAGES="cmake python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
 export GRIDCOIN_CONFIG="--enable-reduce-exports --with-gui=qt5"

cd/06_script_b.sh

@@ -12,7 +12,11 @@ if [[ $HOST = *-apple-* ]]; then
 	KV=$(cat $BASE_ROOT_DIR/depends/hosts/darwin.mk | grep "OSX_MIN_VERSION=")
 	VER=${KV#OSX_MIN_VERSION=}
 	for f in /tmp/release/*.dmg; do
-		mv $f ${f%.dmg}-min-$VER.dmg
+ 		if [[ $HOST = x86_64-* ]]; then
+			mv $f ${f%.dmg}-min-$VER.dmg
+		else
+			mv $f ${f%.dmg}_arm64-min-$VER.dmg
+		fi
 	done
 fi
 

ci/test/00_setup_env_arm.sh

@@ -11,7 +11,7 @@ export HOST=arm-linux-gnueabihf
 # If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
 if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi
 export DPKG_ADD_ARCH="armhf"
-export PACKAGES="g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
+export PACKAGES="g++-arm-linux-gnueabihf busybox cmake libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
 if [ -n "$QEMU_USER_CMD" ]; then
   # Likely cross-compiling, so install the needed gcc and qemu-user
   export PACKAGES="$PACKAGES qemu-user"

ci/test/00_setup_env_linux_i386.sh

@@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_linux_i386
 export DOCKER_NAME_TAG=ubuntu:20.04
 export HOST=i686-pc-linux-gnu
-export PACKAGES="python3 g++-multilib"
+export PACKAGES="cmake python3 g++-multilib"
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=false
 # export RUN_SECURITY_TESTS="true"

ci/test/00_setup_env_mac.sh

@@ -7,11 +7,11 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos_cross
-export DOCKER_NAME_TAG=ubuntu:20.04  # Check that Focal can cross-compile to macos
+export DOCKER_NAME_TAG=ubuntu:24.04
 export HOST=x86_64-apple-darwin
-export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 python3-dev python3-setuptools xorriso"
-export XCODE_VERSION=12.1
-export XCODE_BUILD_ID=12A7403
+export PACKAGES="clang lld llvm cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso"
+export XCODE_VERSION=15.0
+export XCODE_BUILD_ID=15A240d
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"

ci/test/00_setup_env_mac_arm64_cross.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_macos_arm64_cross
+export DOCKER_NAME_TAG=ubuntu:24.04
+export HOST=arm64-apple-darwin
+export PACKAGES="clang lld llvm cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso"
+export XCODE_VERSION=15.0
+export XCODE_BUILD_ID=15A240d
+export RUN_UNIT_TESTS=false
+export RUN_FUNCTIONAL_TESTS=false
+export GOAL="deploy"
+export GRIDCOIN_CONFIG="--with-gui --enable-reduce-exports"

ci/test/00_setup_env_native_tsan.sh

@@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_tsan
 export DOCKER_NAME_TAG=ubuntu:22.04
-export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev"
+export PACKAGES="cmake clang-13 llvm-13 libc++abi-13-dev libc++-13-dev"
 export DEP_OPTS="CC=clang-13 CXX='clang++-13 -stdlib=libc++'"
 export GOAL="install"
 export GRIDCOIN_CONFIG="CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread CC=clang-13 CXX='clang++-13 -stdlib=libc++'"

ci/test/00_setup_env_win32.sh

@@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_win32
 export DOCKER_NAME_TAG=ubuntu:20.04  # Check that focal can cross-compile to win32
 export HOST=i686-w64-mingw32
-export PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt winehq-stable"
+export PACKAGES="cmake python3 nsis g++-mingw-w64-i686 wine-binfmt winehq-stable"
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=false
 # export RUN_SECURITY_TESTS="true"

ci/test/00_setup_env_win64.sh

@@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_win64
 export DOCKER_NAME_TAG=ubuntu:20.04  # Check that focal can cross-compile to win64
 export HOST=x86_64-w64-mingw32
-export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine-stable winehq-stable"
+export PACKAGES="cmake python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine-stable winehq-stable"
 export DPKG_ADD_ARCH="i386"
 export GOAL=""
 export GRIDCOIN_CONFIG="--enable-reduce-exports --with-gui=qt5"

configure.ac

@@ -90,6 +90,7 @@ AC_PATH_PROG([GIT], [git])
 AC_PATH_PROG(CCACHE,ccache)
 AC_PATH_PROG(XGETTEXT,xgettext)
 AC_PATH_PROG(HEXDUMP,hexdump)
+AC_PATH_TOOL([OBJDUMP], [objdump])
 AC_PATH_TOOL(READELF, readelf)
 AC_PATH_TOOL(CPPFILT, c++filt)
 AC_PATH_TOOL(OBJCOPY, objcopy)
@@ -559,7 +560,6 @@ case $host in
          *)
            AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil)
            AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool)
-           AC_PATH_TOOL([OTOOL], [otool], otool)
            AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs)
            AC_PATH_PROGS([DMG], [dmg], dmg)
            AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert)

contrib/macdeploy/README.md

@@ -12,44 +12,42 @@ When complete, it will have produced `Gridcoin.dmg`.
 
 ### Step 1: Obtaining `Xcode.app`
 
-Our current macOS SDK
-(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be
-extracted from
-[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
+Our current macOS SDK can be extracted from
+[Xcode_15.xip](https://download.developer.apple.com/Developer_Tools/Xcode_15/Xcode_15.xip).
 An Apple ID is needed to download this.
 
-After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
-archive. This makes the SDK less-trivial to extract on non-macOS machines. One
-approach (tested on Debian Buster) is outlined below:
+The `sha256sum` of the downloaded XIP archive should be `4daaed2ef2253c9661779fa40bfff50655dc7ec45801aba5a39653e7bcdde48e`.
+
+To extract the `.xip` on Linux:
 
 ```bash
 # Install/clone tools needed for extracting Xcode.app
 apt install cpio
 git clone https://github.com/bitcoin-core/apple-sdk-tools.git
 
-# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current
+# Unpack the xip and place the resulting Xcode.app in your current
 # working directory
-python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_15.xip | cpio -d -i
 ```
 
-On macOS the process is more straightforward:
+On macOS:
 
 ```bash
-xip -x Xcode_12.1.xip
+xip -x Xcode_15.xip
 ```
 
-### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
+### Step 2: Generating the SDK tarball from `Xcode.app`
 
-To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run
-the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
-previous stage) as the first argument.
+To generate the SDK, run the script [`gen-sdk`](./gen-sdk) with the
+path to `Xcode.app` (extracted in the previous stage) as the first argument.
 
 ```bash
-# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from
-# the supplied Xcode.app
 ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
 
+The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar.gz`.
+The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c282aa6892f55d`.
+
 ## Cross Platform Building
 
 Download and extract the SDK to depends/SDKs. Then build the dependencies from
@@ -95,17 +93,7 @@ This version of `cctools` has been patched to use the current version of `clang`
 and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`.
 
 To complicate things further, all builds must target an Apple SDK. These SDKs are free to
-download, but not redistributable. To obtain it, register for an Apple Developer Account,
-then download [Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
-
-This file is many gigabytes in size, but most (but not all) of what we need is
-contained only in a single directory:
-
-```bash
-Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
-```
-
-See the SDK Extraction notes above for how to obtain it.
+download, but not redistributable. See the SDK Extraction notes above for how to obtain it.
 
 The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are
 created using these tools. The build process has been designed to avoid including the

contrib/macdeploy/gen-sdk

@@ -47,17 +47,14 @@ def run():
 
     out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)
 
-    xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1")
-    assert xcode_libcxx_dir.is_dir()
-
     if args.out_sdktgz:
         out_sdktgz_path = pathlib.Path(args.out_sdktgz_path)
     else:
         # Construct our own out_sdktgz if not specified on the command line
         out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name))
 
     def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir):
-        """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files'
+        """Add all files in dir_to_add to tarfp, but prepent alt_base_dir to the files'
         names
 
         e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking:
@@ -85,8 +82,6 @@ def run():
             with tarfile.open(mode="w", fileobj=gzf) as tarfp:
                 print("Adding MacOSX SDK {} files...".format(sdk_version))
                 tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
-                print("Adding libc++ headers...")
-                tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name))
     print("Done! Find the resulting gzipped tarball at:")
     print(out_sdktgz_path.resolve())
 

contrib/macdeploy/macdeployqtplus

@@ -79,7 +79,7 @@ class FrameworkInfo(object):
     bundleBinaryDirectory = "Contents/MacOS"
 
     @classmethod
-    def fromOtoolLibraryLine(cls, line: str) -> Optional['FrameworkInfo']:
+    def fromLibraryLine(cls, line: str) -> Optional['FrameworkInfo']:
         # Note: line must be trimmed
         if line == "":
             return None
@@ -90,7 +90,7 @@ class FrameworkInfo(object):
 
         m = cls.reOLine.match(line)
         if m is None:
-            raise RuntimeError(f"otool line could not be parsed: {line}")
+            raise RuntimeError(f"Line could not be parsed: {line}")
 
         path = m.group(1)
 
@@ -122,7 +122,7 @@ class FrameworkInfo(object):
                     break
                 i += 1
             if i == len(parts):
-                raise RuntimeError(f"Could not find .framework or .dylib in otool line: {line}")
+                raise RuntimeError(f"Could not find .framework or .dylib in line: {line}")
 
             info.frameworkName = parts[i]
             info.frameworkDirectory = "/".join(parts[:i])
@@ -184,24 +184,24 @@ class DeploymentInfo(object):
         return False
 
 def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]:
+    objdump = os.getenv("OBJDUMP", "objdump")
     if verbose:
-        print(f"Inspecting with otool: {binaryPath}")
-    otoolbin=os.getenv("OTOOL", "otool")
-    otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, universal_newlines=True)
-    if otool.returncode != 0:
-        sys.stderr.write(otool.stderr)
+        print(f"Inspecting with {objdump}: {binaryPath}")
+    output = run([objdump, "--macho", "--dylibs-used", binaryPath], stdout=PIPE, stderr=PIPE, text=True)
+    if output.returncode != 0:
+        sys.stderr.write(output.stderr)
         sys.stderr.flush()
-        raise RuntimeError(f"otool failed with return code {otool.returncode}")
+        raise RuntimeError(f"{objdump} failed with return code {output.returncode}")
 
-    otoolLines = otool.stdout.split("\n")
-    otoolLines.pop(0) # First line is the inspected binary
+    lines = output.stdout.split("\n")
+    lines.pop(0) # First line is the inspected binary
     if ".framework" in binaryPath or binaryPath.endswith(".dylib"):
-        otoolLines.pop(0) # Frameworks and dylibs list themselves as a dependency.
+        lines.pop(0) # Frameworks and dylibs list themselves as a dependency.
 
     libraries = []
-    for line in otoolLines:
+    for line in lines:
         line = line.replace("@loader_path", os.path.dirname(binaryPath))
-        info = FrameworkInfo.fromOtoolLibraryLine(line.strip())
+        info = FrameworkInfo.fromLibraryLine(line.strip())
         if info is not None:
             if verbose:
                 print("Found framework:")

depends/Makefile

@@ -1,5 +1,9 @@
 .NOTPARALLEL :
 
+# Pattern rule to print variables, e.g. make print-top_srcdir
+print-%: FORCE
+	@echo '$*'='$($*)'
+
 SOURCES_PATH ?= $(BASEDIR)/sources
 BASE_CACHE ?= $(BASEDIR)/built
 SDK_PATH ?= $(BASEDIR)/SDKs
@@ -8,6 +12,9 @@ NO_WALLET ?=
 NO_UPNP ?=
 FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources
 
+C_STANDARD ?= gnu11
+CXX_STANDARD ?= c++17
+
 BUILD = $(shell ./config.guess)
 HOST ?= $(BUILD)
 PATCHES_PATH = $(BASEDIR)/patches
@@ -145,6 +152,8 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@RANLIB@|$(host_RANLIB)|' \
             -e 's|@NM@|$(host_NM)|' \
             -e 's|@STRIP@|$(host_STRIP)|' \
+            -e 's|@OBJDUMP@|$(host_OBJDUMP)|' \
+            -e 's|@DSYMUTIL@|$(host_DSYMUTIL)|' \
             -e 's|@build_os@|$(build_os)|' \
             -e 's|@host_os@|$(host_os)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \
@@ -196,4 +205,5 @@ download-win:
 download: download-osx download-linux download-win
 
 .PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources
+.PHONY: FORCE
 $(V).SILENT:

depends/README.md

@@ -60,6 +60,8 @@ The following can be set when running make: make FOO=bar
     BASE_CACHE: built packages will be placed here
     SDK_PATH: Path where sdk's can be found (used by macOS)
     FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up
+    C_STANDARD: Set the C standard version used. Defaults to `c11`.
+    CXX_STANDARD: Set the C++ standard version used. Defaults to `c++17`.
     NO_QT: Don't download/build/cache qt and its dependencies
     NO_WALLET: Don't download/build/cache libs needed to enable the wallet
     NO_UPNP: Don't download/build/cache packages needed for enabling upnp