From be9269a44320a7d33d76415761d67ce386b0c58c Mon Sep 17 00:00:00 2001 From: Viktor Kleen Date: Wed, 6 Sep 2023 09:14:59 +0000 Subject: [PATCH] Fix the release artifacts workflow (#1573) * Fix a typo in the release workflow This changes `os.matrix` into the correct `matrix.os`. * Use `nodeRuntimes` as required by nixpkgs master, now * Add nix settings to github-runner.nix * Disable diffutils tests on arm64 musl * Add dependencies on `start-runner` in `release-artifacts.yaml` * Use `--log-format raw-with-logs` like in the CI workflow * Use the correct cargo package for `nickel-static` * Adjust linker flags on arm64 musl * Add github cli and docker to `github-runner.nix` * Combine static binary and docker image building jobs for caching * Allow docker access for github jobs * Use `docker buildx` to assemble the multiplatform image --- .github/workflows/release-artifacts.yaml | 63 +++++++----------------- flake.nix | 21 ++++++-- infra/github-runner.nix | 15 ++++++ 3 files changed, 52 insertions(+), 47 deletions(-) diff --git a/.github/workflows/release-artifacts.yaml b/.github/workflows/release-artifacts.yaml index f48a965167..2438bee85c 100644 --- a/.github/workflows/release-artifacts.yaml +++ b/.github/workflows/release-artifacts.yaml @@ -52,7 +52,6 @@ jobs: needs: - start-runner - docker-multiplatform-image - - static-binary steps: - uses: aws-actions/configure-aws-credentials@v3 with: @@ -76,9 +75,9 @@ jobs: --payload '{"instance_id":"${{ needs.start-runner.outputs.instance_id }}"}' \ response.json cat response.json - - docker-image: - name: "Build docker image" + + release-artifacts: + name: "Build Nickel binary and Docker image" strategy: matrix: os: @@ -87,6 +86,8 @@ jobs: - runs-on: [EC2, ARM64, Linux] architecture: arm64 runs-on: ${{ matrix.os.runs-on }} + needs: + - start-runner steps: - uses: actions/checkout@v4 with: @@ -98,10 +99,20 @@ jobs: experimental-features = nix-command flakes accept-flake-config = true nix_path: "nixpkgs=channel:nixos-unstable" + - name: "Build static binary" + run: | + nix build --log-format raw-with-logs .#nickel-static + cp ./result/bin/nickel nickel-${{ matrix.os.architecture }}-linux + - name: "Upload static binary as release asset" + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.inputs.release_tag }} + run: | + gh release upload --clobber $RELEASE_TAG nickel-${{ matrix.os.architecture }}-linux - id: build-image name: "Build docker image" run: | - nix build --print-build-logs .#dockerImage + nix build --log-format raw-with-logs .#dockerImage cp ./result nickel-${{ matrix.os.architecture }}-docker-image.tar.gz echo "imageName=$(nix eval --raw .#dockerImage.imageName)" >> "$GITHUB_OUTPUT" echo "imageTag=$(nix eval --raw .#dockerImage.imageTag)" >> "$GITHUB_OUTPUT" @@ -126,7 +137,7 @@ jobs: docker-multiplatform-image: name: "Assemble multi-platform Docker image" runs-on: ubuntu-latest - needs: docker-image + needs: release-artifacts steps: - name: Log in to registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin @@ -134,41 +145,5 @@ jobs: env: RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.inputs.release_tag }} run: | - docker manifest create \ - ghcr.io/tweag/nickel:$RELEASE_TAG \ - --amend ghcr.io/tweag/nickel:$RELEASE_TAG-x86_64 \ - --amend ghcr.io/tweag/nickel:$RELEASE_TAG-arm64 \ - docker manifest push ghcr.io/tweag/nickel:$RELEASE_TAG - - - static-binary: - name: "Build Nickel release binary" - strategy: - matrix: - os: - - runs-on: ubuntu-latest - architecture: x86_64 - - runs-on: [EC2, ARM64, Linux] - architecture: arm64 - runs-on: ${{ matrix.os.runs-on }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'release' && '' || github.event.inputs.release_tag }} - - uses: cachix/install-nix-action@v23 - name: "Installing Nix" - with: - extra_nix_config: | - experimental-features = nix-command flakes - accept-flake-config = true - nix_path: "nixpkgs=channel:nixos-unstable" - - name: "Build static binary" - run: | - nix build --print-build-logs .#nickel-static - cp ./result/bin/nickel nickel-${{ os.matrix.architecture }}-linux - - name: "Upload static binary as release asset" - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.inputs.release_tag }} - run: | - gh release upload --clobber $RELEASE_TAG nickel-${{ os.matrix.architecture }}-linux + docker buildx imagetools create -t ghcr.io/tweag/nickel:$RELEASE_TAG ghcr.io/tweag/nickel:$RELEASE_TAG-x86_64 ghcr.io/tweag/nickel:$RELEASE_TAG-arm64 + docker buildx imagetools inspect ghcr.io/tweag/nickel:$RELEASE_TAG diff --git a/flake.nix b/flake.nix index 980b0cadde..1cd48b8312 100644 --- a/flake.nix +++ b/flake.nix @@ -63,6 +63,18 @@ inherit system; overlays = [ (import rust-overlay) + # gnulib tests in diffutils fail for musl arm64, cf. https://github.com/NixOS/nixpkgs/pull/241281 + (final: prev: { + diffutils = + if !(final.stdenv.hostPlatform.isMusl && final.stdenv.hostPlatform.isAarch64) then + prev.diffutils + else + prev.diffutils.overrideAttrs (old: { + postPatch = '' + sed -i 's:gnulib-tests::g' Makefile.in + ''; + }); + }) ]; }; @@ -222,7 +234,7 @@ NICKEL_NIX_BUILD_REV = self.shortRev or "dirty"; }; - buildPackage = { pnameSuffix, extraBuildArgs ? "", extraArgs ? { } }: + buildPackage = { pnameSuffix, cargoPackage ? "${pname}${pnameSuffix}", extraBuildArgs ? "", extraArgs ? { } }: craneLib.buildPackage ({ inherit pname @@ -232,7 +244,7 @@ cargoArtifacts env; - cargoExtraArgs = "${cargoBuildExtraArgs} ${extraBuildArgs} --package ${pname}${pnameSuffix}"; + cargoExtraArgs = "${cargoBuildExtraArgs} ${extraBuildArgs} --package ${cargoPackage}"; } // extraArgs); in rec { @@ -250,6 +262,7 @@ # libc and clang with libc++ to build C and C++ dependencies. We # tried building with libstdc++ but without success. buildPackage { + cargoPackage = "nickel-lang-cli"; pnameSuffix = "-static"; extraArgs = { CARGO_BUILD_TARGET = pkgs.rust.toRustTarget pkgs.pkgsMusl.stdenv.hostPlatform; @@ -258,7 +271,9 @@ # them explicitly. Also, `libcxx` expects to be linked with # `libcxxabi` at the end, and we need to make the rust linker # aware of that. - RUSTFLAGS = "-L${pkgs.pkgsMusl.llvmPackages.libcxx}/lib -L${pkgs.pkgsMusl.llvmPackages.libcxxabi}/lib -lstatic=c++abi"; + # + # We also explicitly add `libc` because of https://github.com/rust-lang/rust/issues/89626. + RUSTFLAGS = "-L${pkgs.pkgsMusl.llvmPackages.libcxx}/lib -L${pkgs.pkgsMusl.llvmPackages.libcxxabi}/lib -lstatic=c++abi -C link-arg=-lc"; # Explain to `cc-rs` that it should use the `libcxx` C++ # standard library, and a static version of it, when building # C++ libraries. The `cc-rs` crate is typically used in diff --git a/infra/github-runner.nix b/infra/github-runner.nix index 8797e72976..63db5e9412 100644 --- a/infra/github-runner.nix +++ b/infra/github-runner.nix @@ -25,7 +25,10 @@ in }; }); }; + nodeRuntimes = [ "node16" "node20" ]; extraPackages = with pkgs; [ + gh + docker gawk nix ]; @@ -34,6 +37,18 @@ in extraLabels = [ "EC2" ]; + serviceOverrides = { + Group = "docker"; + }; + }; + + virtualisation.docker.enable = true; + + nix.settings = { + experimental-features = [ "nix-command" "flakes" ]; + substituters = [ "https://tweag-nickel.cachix.org" ]; + trusted-public-keys = [ "tweag-nickel.cachix.org-1:GIthuiK4LRgnW64ALYEoioVUQBWs0jexyoYVeLDBwRA=" ]; + accept-flake-config = true; }; systemd.services.github-runner-init = {