Merge pull request #283 from subspace/ci-fix

add automake and hwloc for macOS and support numa CI workflow

.github/workflows/ci-tests.yml

@@ -16,6 +16,8 @@ on:
         required: true
         default: false
         type: boolean
+  merge_group:
+    types: [checks_requested]
 
 env:
   # Not needed in CI, should make things a bit faster
@@ -32,39 +34,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build:
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
-            target: x86_64-unknown-linux-gnu
-            suffix: ubuntu-x86_64-skylake-${{ github.ref_name }}
-            rustflags: "-C target-cpu=skylake"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
-            target: x86_64-unknown-linux-gnu
-            suffix: ubuntu-x86_64-v2-${{ github.ref_name }}
-            rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
-            target: aarch64-unknown-linux-gnu
-            suffix: ubuntu-aarch64-${{ github.ref_name }}
-            # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to
-            #  at least 1.61: https://github.com/RustCrypto/block-ciphers/issues/373
-            rustflags: "-C linker=aarch64-linux-gnu-gcc --cfg aes_armv8"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || '"macos-12"') }}
-            target: aarch64-apple-darwin
-            suffix: macos-aarch64-${{ github.ref_name }}
-            # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to
-            #  at least 1.61: https://github.com/RustCrypto/block-ciphers/issues/373
-            rustflags: "--cfg aes_armv8"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || '"macos-12"') }}
-            target: x86_64-apple-darwin
-            suffix: macos-x86_64-${{ github.ref_name }}
-            rustflags: ""
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }}
-            target: x86_64-pc-windows-msvc
-            suffix: windows-x86_64-skylake-${{ github.ref_name }}
-            rustflags: "-C target-cpu=skylake"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }}
-            target: x86_64-pc-windows-msvc
-            suffix: windows-x86_64-v2-${{ github.ref_name }}
-            rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes"
+        os: ${{ fromJson(github.repository_owner == 'subspace' && '[["self-hosted", "ubuntu-20.04-x86-64"], ["self-hosted", "macos-12-arm64"], ["self-hosted", "windows-server-2022-x86-64"]]' || '["ubuntu-20.04", "macos-12", "windows-2022"]') }}
         run-all:
           - ${{ inputs.test-macos-and-windows == true || github.ref == 'refs/heads/main' }}
         exclude: # exclude macos-12 and windows-2022 when the condition is false
@@ -73,7 +43,7 @@ jobs:
           - run-all: false
             os: windows-2022
 
-    runs-on: ${{ matrix.build.os }}
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0
@@ -102,52 +72,30 @@ jobs:
         # May not exist on self-hosted runners
         continue-on-error: true
 
-      - name: AArch64 cross-compile packages
-        run: |
-          FLAVOR="$(lsb_release -sc)"
-
-          sudo tee /etc/apt/sources.list.d/arm64.list <<LIST
-            deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} main restricted
-            deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates main restricted
-            deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} universe
-            deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates universe
-            deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} multiverse
-            deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates multiverse
-            deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-backports main restricted universe multiverse
-          LIST
-          sudo sed -i 's/deb http/deb [arch=amd64] http/' /etc/apt/sources.list
-          # GitHub runners use mirror file
-          sudo sed -i 's/deb mirror/deb [arch=amd64] mirror/' /etc/apt/sources.list
-
-          sudo dpkg --add-architecture arm64
-          sudo apt-get update
-          # zlib1g-dev:arm64 is only necessary because amd64 version is present on the host and cross-compilation of
-          # hwlocality-sys fails otherwise
-          sudo apt-get install -y --no-install-recommends \
-            g++-aarch64-linux-gnu \
-            gcc-aarch64-linux-gnu \
-            libc6-dev-arm64-cross \
-            zlib1g-dev:arm64
-
-          echo "PKG_CONFIG_ALLOW_CROSS=true" >> $GITHUB_ENV
-        if: matrix.build.target == 'aarch64-unknown-linux-gnu'
-
       - name: cargo fmt
         uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # @v1.0.1
         with:
           command: fmt
           args: --all -- --check
 
-      - name: cargo test --locked
+      - name: cargo test
         uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # @v1.0.1
         with:
           command: test
 
-      - name: cargo clippy --locked
+      - name: cargo clippy --locked (Linux & Windows)
         uses: actions-rs/clippy-check@b5b5f21f4797c02da247df37026fcd0a5024aa4d # @v1.0.7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --locked --all-targets -- -D warnings
+        if: runner.os != 'macOS'
+
+      - name: cargo clippy --locked (macOS)
+        uses: actions-rs/clippy-check@b5b5f21f4797c02da247df37026fcd0a5024aa4d # @v1.0.7
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --locked --all-targets --no-default-features -- -D warnings
+        if: runner.os == 'macOS'
 
       - name: Check Documentation
         run: cargo doc --locked --all --no-deps

.github/workflows/release.yml

@@ -17,42 +17,38 @@ jobs:
     strategy:
       matrix:
         build:
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }}
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
             target: x86_64-unknown-linux-gnu
-            production_target: target/x86_64-unknown-linux-gnu/production
-            suffix: ubuntu-x86_64-v2-${{ github.ref_name }}
-            rustflags: "-C target-cpu=x86-64-v2"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }}
-            target: x86_64-unknown-linux-gnu
-            production_target: target/x86_64-unknown-linux-gnu/production
             suffix: ubuntu-x86_64-skylake-${{ github.ref_name }}
             rustflags: "-C target-cpu=skylake"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-20.04') }}
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
+            target: x86_64-unknown-linux-gnu
+            suffix: ubuntu-x86_64-v2-${{ github.ref_name }}
+            rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes"
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
             target: aarch64-unknown-linux-gnu
-            production_target: target/aarch64-unknown-linux-gnu/aarch64linux
             suffix: ubuntu-aarch64-${{ github.ref_name }}
-            rustflags: "-C linker=aarch64-linux-gnu-gcc"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || 'macos-12') }}
-            target: x86_64-apple-darwin
-            production_target: target/x86_64-apple-darwin/production
-            suffix: macos-x86_64-${{ github.ref_name }}
-            rustflags: ""
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || 'macos-12') }}
+            # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to
+            #  at least 1.61: https://github.com/RustCrypto/block-ciphers/issues/373
+            rustflags: "-C linker=aarch64-linux-gnu-gcc --cfg aes_armv8"
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || '"macos-12"') }}
             target: aarch64-apple-darwin
-            production_target: target/aarch64-apple-darwin/production
             suffix: macos-aarch64-${{ github.ref_name }}
+            # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate bumps MSRV to
+            #  at least 1.61: https://github.com/RustCrypto/block-ciphers/issues/373
+            rustflags: "--cfg aes_armv8"
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "macos-12-arm64"]' || '"macos-12"') }}
+            target: x86_64-apple-darwin
+            suffix: macos-x86_64-${{ github.ref_name }}
             rustflags: ""
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || 'windows-2022') }}
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }}
             target: x86_64-pc-windows-msvc
-            production_target: target/x86_64-pc-windows-msvc/production
-            suffix: windows-x86_64-v2-${{ github.ref_name }}
-            rustflags: "-C target-cpu=x86-64-v2"
-          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || 'windows-2022') }}
-            target: x86_64-pc-windows-msvc
-            production_target: target/x86_64-pc-windows-msvc/production
             suffix: windows-x86_64-skylake-${{ github.ref_name }}
             rustflags: "-C target-cpu=skylake"
-
+          - os: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }}
+            target: x86_64-pc-windows-msvc
+            suffix: windows-x86_64-v2-${{ github.ref_name }}
+            rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes"
     runs-on: ${{ matrix.build.os }}
 
     env:
@@ -119,7 +115,7 @@ jobs:
 
       - name: Build the executable (other than aarch64 linux)
         uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # @v1.0.1
-        if: matrix.build.target != 'aarch64-unknown-linux-gnu'
+        if: matrix.build.target != 'aarch64-unknown-linux-gnu' && matrix.build.os != 'macOS'
         with:
           command: build
           args: --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin pulsar
@@ -132,6 +128,14 @@ jobs:
           command: build
           args: --locked -Z build-std --target ${{ matrix.build.target }} --profile aarch64linux --bin pulsar
 
+      # We build macOS without `numa` feature, primarily because of https://github.com/HadrienG2/hwlocality/issues/31
+      - name: Build the executable (macOS)
+        uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # @v1.0.1
+        with:
+          command: build
+          args: --locked -Z build-std --target ${{ matrix.build.target }} --profile production --no-default-features --bin pulsar
+        if: runner.os == 'macOS'
+
       - name: Sign and Notarize Application (macOS)
         run: |
           echo "Importing certificate"