Fix the release artifacts workflow (#1573)

* Fix a typo in the release workflow This changes `os.matrix` into the correct `matrix.os`. * Use `nodeRuntimes` as required by nixpkgs master, now * Add nix settings to github-runner.nix * Disable diffutils tests on arm64 musl * Add dependencies on `start-runner` in `release-artifacts.yaml` * Use `--log-format raw-with-logs` like in the CI workflow * Use the correct cargo package for `nickel-static` * Adjust linker flags on arm64 musl * Add github cli and docker to `github-runner.nix` * Combine static binary and docker image building jobs for caching * Allow docker access for github jobs * Use `docker buildx` to assemble the multiplatform image
tweag · Sep 6, 2023 · be9269a · be9269a
1 parent c4bda49
commit be9269a
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 47 deletions.
diff --git a/.github/workflows/release-artifacts.yaml b/.github/workflows/release-artifacts.yaml
@@ -52,7 +52,6 @@ jobs:
     needs:
       - start-runner
       - docker-multiplatform-image
-      - static-binary
     steps:
       - uses: aws-actions/configure-aws-credentials@v3
         with:
@@ -76,9 +75,9 @@ jobs:
             --payload '{"instance_id":"${{ needs.start-runner.outputs.instance_id }}"}' \
             response.json
           cat response.json
-  
-  docker-image:
-    name: "Build docker image"
+
+  release-artifacts:
+    name: "Build Nickel binary and Docker image"
     strategy:
       matrix:
         os:
@@ -87,6 +86,8 @@ jobs:
           - runs-on: [EC2, ARM64, Linux]
             architecture: arm64
     runs-on: ${{ matrix.os.runs-on }}
+    needs:
+      - start-runner
     steps:
       - uses: actions/checkout@v4
         with:
@@ -98,10 +99,20 @@ jobs:
             experimental-features = nix-command flakes
             accept-flake-config = true
           nix_path: "nixpkgs=channel:nixos-unstable"
+      - name: "Build static binary"
+        run: |
+          nix build --log-format raw-with-logs .#nickel-static
+          cp ./result/bin/nickel nickel-${{ matrix.os.architecture }}-linux
+      - name: "Upload static binary as release asset"
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.inputs.release_tag }}
+        run: |
+          gh release upload --clobber $RELEASE_TAG nickel-${{ matrix.os.architecture }}-linux
       - id: build-image
         name: "Build docker image"
         run: |
-          nix build --print-build-logs .#dockerImage
+          nix build --log-format raw-with-logs .#dockerImage
           cp ./result nickel-${{ matrix.os.architecture }}-docker-image.tar.gz
           echo "imageName=$(nix eval --raw .#dockerImage.imageName)" >> "$GITHUB_OUTPUT"
           echo "imageTag=$(nix eval --raw .#dockerImage.imageTag)" >> "$GITHUB_OUTPUT"
@@ -126,49 +137,13 @@ jobs:
   docker-multiplatform-image:
     name: "Assemble multi-platform Docker image"
     runs-on: ubuntu-latest
-    needs: docker-image
+    needs: release-artifacts
     steps:
       - name: Log in to registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
       - name: Assemble and push image
         env:
           RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.inputs.release_tag }}
         run: |
-          docker manifest create \
-            ghcr.io/tweag/nickel:$RELEASE_TAG \
-            --amend ghcr.io/tweag/nickel:$RELEASE_TAG-x86_64 \
-            --amend ghcr.io/tweag/nickel:$RELEASE_TAG-arm64 \
-          docker manifest push ghcr.io/tweag/nickel:$RELEASE_TAG
-        
-
-  static-binary:
-    name: "Build Nickel release binary"
-    strategy:
-      matrix:
-        os:
-          - runs-on: ubuntu-latest
-            architecture: x86_64
-          - runs-on: [EC2, ARM64, Linux]
-            architecture: arm64
-    runs-on: ${{ matrix.os.runs-on }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event_name == 'release' && '' || github.event.inputs.release_tag }}
-      - uses: cachix/install-nix-action@v23
-        name: "Installing Nix"
-        with:
-          extra_nix_config: |
-            experimental-features = nix-command flakes
-            accept-flake-config = true
-          nix_path: "nixpkgs=channel:nixos-unstable"
-      - name: "Build static binary"
-        run: |
-          nix build --print-build-logs .#nickel-static
-          cp ./result/bin/nickel nickel-${{ os.matrix.architecture }}-linux
-      - name: "Upload static binary as release asset"
-        env:
-          GH_TOKEN: ${{ github.token }}
-          RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.inputs.release_tag }}
-        run: |
-          gh release upload --clobber $RELEASE_TAG nickel-${{ os.matrix.architecture }}-linux
+          docker buildx imagetools create -t ghcr.io/tweag/nickel:$RELEASE_TAG ghcr.io/tweag/nickel:$RELEASE_TAG-x86_64 ghcr.io/tweag/nickel:$RELEASE_TAG-arm64
+          docker buildx imagetools inspect ghcr.io/tweag/nickel:$RELEASE_TAG 
diff --git a/flake.nix b/flake.nix
@@ -63,6 +63,18 @@
         inherit system;
         overlays = [
           (import rust-overlay)
+          # gnulib tests in diffutils fail for musl arm64, cf. https://github.com/NixOS/nixpkgs/pull/241281
+          (final: prev: {
+            diffutils =
+              if !(final.stdenv.hostPlatform.isMusl && final.stdenv.hostPlatform.isAarch64) then
+                prev.diffutils
+              else
+                prev.diffutils.overrideAttrs (old: {
+                  postPatch = ''
+                    sed -i 's:gnulib-tests::g' Makefile.in
+                  '';
+                });
+          })
         ];
       };
 
@@ -222,7 +234,7 @@
             NICKEL_NIX_BUILD_REV = self.shortRev or "dirty";
           };
 
-          buildPackage = { pnameSuffix, extraBuildArgs ? "", extraArgs ? { } }:
+          buildPackage = { pnameSuffix, cargoPackage ? "${pname}${pnameSuffix}", extraBuildArgs ? "", extraArgs ? { } }:
             craneLib.buildPackage ({
               inherit
                 pname
@@ -232,7 +244,7 @@
                 cargoArtifacts
                 env;
 
-              cargoExtraArgs = "${cargoBuildExtraArgs} ${extraBuildArgs} --package ${pname}${pnameSuffix}";
+              cargoExtraArgs = "${cargoBuildExtraArgs} ${extraBuildArgs} --package ${cargoPackage}";
             } // extraArgs);
         in
         rec {
@@ -250,6 +262,7 @@
             # libc and clang with libc++ to build C and C++ dependencies. We
             # tried building with libstdc++ but without success.
               buildPackage {
+                cargoPackage = "nickel-lang-cli";
                 pnameSuffix = "-static";
                 extraArgs = {
                   CARGO_BUILD_TARGET = pkgs.rust.toRustTarget pkgs.pkgsMusl.stdenv.hostPlatform;
@@ -258,7 +271,9 @@
                   # them explicitly. Also, `libcxx` expects to be linked with
                   # `libcxxabi` at the end, and we need to make the rust linker
                   # aware of that.
-                  RUSTFLAGS = "-L${pkgs.pkgsMusl.llvmPackages.libcxx}/lib -L${pkgs.pkgsMusl.llvmPackages.libcxxabi}/lib -lstatic=c++abi";
+                  #
+                  # We also explicitly add `libc` because of https://github.com/rust-lang/rust/issues/89626.
+                  RUSTFLAGS = "-L${pkgs.pkgsMusl.llvmPackages.libcxx}/lib -L${pkgs.pkgsMusl.llvmPackages.libcxxabi}/lib -lstatic=c++abi -C link-arg=-lc";
                   # Explain to `cc-rs` that it should use the `libcxx` C++
                   # standard library, and a static version of it, when building
                   # C++ libraries. The `cc-rs` crate is typically used in

diff --git a/infra/github-runner.nix b/infra/github-runner.nix
@@ -25,7 +25,10 @@ in
           };
       });
     };
+    nodeRuntimes = [ "node16" "node20" ];
     extraPackages = with pkgs; [
+      gh
+      docker
       gawk
       nix
     ];
@@ -34,6 +37,18 @@ in
     extraLabels = [
       "EC2"
     ];
+    serviceOverrides = {
+      Group = "docker";
+    };
+  };
+
+  virtualisation.docker.enable = true;
+
+  nix.settings = {
+    experimental-features = [ "nix-command" "flakes" ];
+    substituters = [ "https://tweag-nickel.cachix.org" ];
+    trusted-public-keys = [ "tweag-nickel.cachix.org-1:GIthuiK4LRgnW64ALYEoioVUQBWs0jexyoYVeLDBwRA=" ];
+    accept-flake-config = true;
   };
 
   systemd.services.github-runner-init = {