From e1af4339bb108627bb7ac6df9b681a8ea384b669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A7=91=F0=9F=8F=BB=E2=80=8D=F0=9F=92=BB=20Romain=20M?= =?UTF-8?q?arcadier?= Date: Thu, 27 Jul 2023 12:14:10 +0200 Subject: [PATCH] feat(superchain): multi-build on debian base image Build on several base debian iages (10/buster, 11/bullseye, 12/bookworm) and publish separately. --- .github/workflows/docker-images.yml | 80 ++++++++++++++++----------- superchain/Dockerfile | 83 ++++++++++++++++------------- superchain/README.md | 6 ++- superchain/build-local.sh | 14 +++-- 4 files changed, 111 insertions(+), 72 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index c498d29312..04129f2c1a 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -21,6 +21,10 @@ jobs: strategy: fail-fast: false matrix: + debian: + - 'buster' # 10 + - 'bullseye' # 11 + - 'bookworm' # 12 node: ['14', '16', '18', '20'] env: # Node version whose images will be aliased without the -nodeXX segment @@ -139,6 +143,7 @@ jobs: --pull \ --build-arg BUILD_TIMESTAMP="${{ steps.build-time.outputs.value }}" \ --build-arg COMMIT_ID='${{ github.sha }}' \ + --build-arg DEBIAN_VERSION=${{ matrix.debian }} \ --build-arg NODE_MAJOR_VERSION=${{ matrix.node }} \ -f superchain/Dockerfile \ . @@ -152,6 +157,7 @@ jobs: --target superchain \ --build-arg BUILD_TIMESTAMP="${{ steps.build-time.outputs.value }}" \ --build-arg COMMIT_ID='${{ github.sha }}' \ + --build-arg DEBIAN_VERSION=${{ matrix.debian }} \ --build-arg NODE_MAJOR_VERSION=${{ matrix.node }} \ -f superchain/Dockerfile \ . @@ -184,11 +190,12 @@ jobs: --push \ --build-arg BUILD_TIMESTAMP="${{ steps.build-time.outputs.value }}" \ --build-arg COMMIT_ID='${{ github.sha }}' \ + --build-arg DEBIAN_VERSION=${{ matrix.debian }} \ --build-arg NODE_MAJOR_VERSION=${{ matrix.node }} \ - --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-buster-slim-nightly" \ - --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-buster-slim-node${{ matrix.node }}-nightly" \ - --tag "jsii/superchain:1-buster-slim-nightly" \ - --tag "jsii/superchain:1-buster-slim-node${{ matrix.node }}-nightly" \ + --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-${{ matrix.debian }}-slim-nightly" \ + --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-${{ matrix.debian }}-slim-node${{ matrix.node }}-nightly" \ + --tag "jsii/superchain:1-${{ matrix.debian }}-slim-nightly" \ + --tag "jsii/superchain:1-${{ matrix.debian }}-slim-node${{ matrix.node }}-nightly" \ -f superchain/Dockerfile \ . else @@ -199,9 +206,10 @@ jobs: --push \ --build-arg BUILD_TIMESTAMP="${{ steps.build-time.outputs.value }}" \ --build-arg COMMIT_ID='${{ github.sha }}' \ + --build-arg DEBIAN_VERSION=${{ matrix.debian }} \ --build-arg NODE_MAJOR_VERSION=${{ matrix.node }} \ - --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-buster-slim-node${{ matrix.node }}-nightly" \ - --tag "jsii/superchain:1-buster-slim-node${{ matrix.node }}-nightly" \ + --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-${{ matrix.debian }}-slim-node${{ matrix.node }}-nightly" \ + --tag "jsii/superchain:1-${{ matrix.debian }}-slim-node${{ matrix.node }}-nightly" \ -f superchain/Dockerfile \ . fi @@ -212,31 +220,43 @@ jobs: run: |- # If the current version is the default version, also tag this with the unqualified ':1-*' label if [[ "${{ matrix.node }}" == "$DEFAULT_NODE_MAJOR_VERSION" ]]; then - docker buildx build \ - --builder ${{ steps.buildx.outputs.name }} \ - --platform linux/amd64,linux/arm64 \ - --target superchain \ - --push \ - --build-arg BUILD_TIMESTAMP="${{ steps.build-time.outputs.value }}" \ - --build-arg COMMIT_ID='${{ github.sha }}' \ - --build-arg NODE_MAJOR_VERSION=${{ matrix.node }} \ - --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-buster-slim" \ - --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-buster-slim-node${{ matrix.node }}" \ - --tag "jsii/superchain:1-buster-slim" \ - --tag "jsii/superchain:1-buster-slim-node${{ matrix.node }}" \ - -f superchain/Dockerfile \ + docker buildx build \ + --builder ${{ steps.buildx.outputs.name }} \ + --platform linux/amd64,linux/arm64 \ + --target superchain \ + --push \ + --build-arg BUILD_TIMESTAMP="${{ steps.build-time.outputs.value }}" \ + --build-arg COMMIT_ID='${{ github.sha }}' \ + --build-arg DEBIAN_VERSION=${{ matrix.debian }} \ + --build-arg NODE_MAJOR_VERSION=${{ matrix.node }} \ + --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-${{ matrix.debian }}-slim" \ + --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-${{ matrix.debian }}-slim-node${{ matrix.node }}" \ + --tag "jsii/superchain:1-${{ matrix.debian }}-slim" \ + --tag "jsii/superchain:1-${{ matrix.debian }}-slim-node${{ matrix.node }}" \ + -f superchain/Dockerfile \ . else - docker buildx build \ - --builder ${{ steps.buildx.outputs.name }} \ - --platform linux/amd64,linux/arm64 \ - --target superchain \ - --push \ - --build-arg BUILD_TIMESTAMP="${{ steps.build-time.outputs.value }}" \ - --build-arg COMMIT_ID='${{ github.sha }}' \ - --build-arg NODE_MAJOR_VERSION=${{ matrix.node }} \ - --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-buster-slim-node${{ matrix.node }}" \ - --tag "jsii/superchain:1-buster-slim-node${{ matrix.node }}" \ - -f superchain/Dockerfile \ + docker buildx build \ + --builder ${{ steps.buildx.outputs.name }} \ + --platform linux/amd64,linux/arm64 \ + --target superchain \ + --push \ + --build-arg BUILD_TIMESTAMP="${{ steps.build-time.outputs.value }}" \ + --build-arg COMMIT_ID='${{ github.sha }}' \ + --build-arg DEBIAN_VERSION=${{ matrix.debian }} \ + --build-arg NODE_MAJOR_VERSION=${{ matrix.node }} \ + --tag "${{ secrets.ECR_PUBLIC_REGISTRY }}:1-${{ matrix.debian }}-slim-node${{ matrix.node }}" \ + --tag "jsii/superchain:1-${{ matrix.debian }}-slim-node${{ matrix.node }}" \ + -f superchain/Dockerfile \ . fi + + done: + name: 'Done' + runs-on: ['ubuntu-latest'] + needs: ['superchain'] + steps: + # This is just a join target to simplify branch protection settings... + - name: 'All done' + run: |- + echo "All done!" diff --git a/superchain/Dockerfile b/superchain/Dockerfile index d6da570f7f..17e7437f42 100644 --- a/superchain/Dockerfile +++ b/superchain/Dockerfile @@ -14,11 +14,12 @@ # more efficient, and allows us to run cross-architecture builds in the most efficient possible manner, by running the # "platform-independent" elements within the build platform, which does not require any kind of emulation. +ARG DEBIAN_VERSION="bookworm" ######################################################################################################################## # Prepare install images of "manual" binary distributions (runs on BUILD platform for speed) ######################################################################################################################## -FROM --platform=${BUILDPLATFORM} public.ecr.aws/debian/debian:bookworm as bindist +FROM --platform=${BUILDPLATFORM} public.ecr.aws/debian/debian:${DEBIAN_VERSION}-slim as bindist # Build & target platforms, they are provided by buildx and will look like "linux/amd64" or "linux/arm64" ARG BUILDPLATFORM @@ -28,12 +29,15 @@ ARG TARGETPLATFORM ENV BUILDPLATFORM=${BUILDPLATFORM:-linux/amd64} \ TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64} +ARG DEBIAN_VERSION="bookworm" + # We require a couple of tools to be available in order to work here... -RUN echo "deb http://deb.debian.org/debian bookworm-backports main" > /etc/apt/sources.list.d/buster-backports.list \ +RUN echo "deb http://deb.debian.org/debian ${DEBIAN_VERSION}-backports main" \ + > "/etc/apt/sources.list.d/${DEBIAN_VERSION}-backports.list" \ && apt-get update \ && apt-get install -y gpg tar zsh \ # We need a "recent" (>= 7.71) version of curl for --retry-all-errors, so we get it from backports... - && apt-get install -y -t bookworm-backports curl + && apt-get install -y -t "${DEBIAN_VERSION}-backports" curl # We'll be using zsh substitutions, so ensuring this is the shell we use SHELL ["/bin/zsh", "-c"] @@ -43,9 +47,9 @@ ARG M2_VERSION="3.9.3" ENV M2_DISTRO="https://www.apache.org/dist/maven/maven-3" RUN set -eo pipefail \ && curl -fSsL "${M2_DISTRO}/${M2_VERSION}/binaries/apache-maven-${M2_VERSION}-bin.tar.gz" \ - -o /tmp/apache-maven.tar.gz \ + -o /tmp/apache-maven.tar.gz \ && curl -fSsL "${M2_DISTRO}/${M2_VERSION}/binaries/apache-maven-${M2_VERSION}-bin.tar.gz.asc" \ - -o /tmp/apache-maven.tar.gz.asc \ + -o /tmp/apache-maven.tar.gz.asc \ && mkdir -p /tmp/gpg-maven && chmod go-rwx /tmp/gpg-maven \ && curl -fSsL "https://www.apache.org/dist/maven/KEYS" | gpg --homedir /tmp/gpg-maven --import \ && gpg --homedir /tmp/gpg-maven --verify /tmp/apache-maven.tar.gz.asc /tmp/apache-maven.tar.gz \ @@ -57,19 +61,18 @@ ARG DOTNET_CHANNEL="6.0" ENV DOTNET_FEED="https://dotnetcli.blob.core.windows.net/dotnet" RUN DOTNET_VERSION=$(curl -fSsL "${DOTNET_FEED}/Sdk/${DOTNET_CHANNEL}/latest.version") \ && DOTNET_ASSET="dotnet-sdk-${DOTNET_VERSION}-linux-${${TARGETPLATFORM#linux/}/amd64/x64}.tar.gz" \ - && curl -fSsL "${DOTNET_FEED}/Sdk/${DOTNET_VERSION}/${DOTNET_ASSET}" \ - -o /tmp/dotnet.tar.gz \ + && curl -fSsL "${DOTNET_FEED}/Sdk/${DOTNET_VERSION}/${DOTNET_ASSET}" -o /tmp/dotnet.tar.gz \ && mkdir -p /opt/microsoft/dotnet \ && tar zxf /tmp/dotnet.tar.gz -C /opt/microsoft/dotnet # Prepare PowerShell LTS distribution ENV POWERSHELL_RELEASES="https://github.com/PowerShell/PowerShell/releases" RUN POWERSHELL_RELEASE=$(curl -X GET -fSsIL "https://aka.ms/powershell-release?tag=lts" -o /dev/null \ - --retry 5 --retry-all-errors -w %{url_effective}) \ + --retry 5 --retry-all-errors -w %{url_effective}) \ && POWERSHELL_VERSION=${POWERSHELL_RELEASE#${POWERSHELL_RELEASES}/tag/v} \ && ASSET="powershell-${POWERSHELL_VERSION}-linux-${${TARGETPLATFORM#linux/}/amd64/x64}.tar.gz" \ && curl -fSsL "${POWERSHELL_RELEASES}/download/v${POWERSHELL_VERSION}/${ASSET}" --retry 5 --retry-all-errors \ - -o /tmp/powershell.tar.gz \ + -o /tmp/powershell.tar.gz \ && mkdir -p /opt/microsoft/powershell \ && tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell \ && chmod +x /opt/microsoft/powershell/pwsh @@ -83,7 +86,7 @@ RUN curl -fSsL "https://golang.org/dl/go${GO_VERSION}.linux-${TARGETPLATFORM#lin ######################################################################################################################## # Set up the image ######################################################################################################################## -FROM public.ecr.aws/debian/debian:bookworm-slim as staging +FROM public.ecr.aws/debian/debian:${DEBIAN_VERSION}-slim as staging # Build & target platforms, they are provided by buildx and will look like "linux/amd64" or "linux/arm64" ARG BUILDPLATFORM @@ -93,6 +96,8 @@ ARG TARGETPLATFORM ENV BUILDPLATFORM=${BUILDPLATFORM:-linux/amd64} \ TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64} +ARG DEBIAN_VERSION="bookworm" + SHELL ["/bin/bash", "-c"] # Set locale and some other interesting environment variables @@ -111,31 +116,33 @@ ENV LANG="C.UTF-8" # Installing the system dependencies we need... RUN apt-get update \ - && apt-get -y install \ - apt-transport-https \ - build-essential \ - ca-certificates \ - curl \ - dirmngr \ - git \ - gnupg \ - gzip \ - libffi-dev \ - libicu72 \ - libssl-dev \ - openssl \ - rsync \ - sudo \ - unzip \ - zip \ - acl \ + && apt-get -y upgrade \ + && apt-get -y install --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + dirmngr \ + git \ + gnupg \ + gzip \ + libffi-dev \ + libicu-dev \ + libssl-dev \ + openssl \ + rsync \ + sudo \ + unzip \ + zip \ + acl \ && rm -rf /var/lib/apt/lists/* # Install mono COPY superchain/gpg/mono.asc /tmp/mono.asc RUN apt-key add /tmp/mono.asc && rm /tmp/mono.asc \ + # Mono only provides a Debian Buster (10) distribution point, but it works with subsequent debians, too... && echo "deb https://download.mono-project.com/repo/debian stable-buster main" \ - > /etc/apt/sources.list.d/mono-official-stable.list \ + > /etc/apt/sources.list.d/mono-official-stable.list \ && apt-get update \ && apt-get -y install mono-devel \ && rm -rf /var/lib/apt/lists/* @@ -180,7 +187,7 @@ RUN apt-key add /tmp/corretto.asc && rm /tmp/corretto.asc # Install Docker COPY superchain/gpg/docker.asc /tmp/docker.asc RUN apt-key add /tmp/docker.asc && rm /tmp/docker.asc \ - && echo "deb https://download.docker.com/linux/debian buster stable" > /etc/apt/sources.list.d/docker.list \ + && echo "deb https://download.docker.com/linux/debian ${DEBIAN_VERSION} stable" > /etc/apt/sources.list.d/docker.list \ && apt-get update \ && apt-get -y install docker-ce docker-ce-cli containerd.io \ && rm -rf /var/lib/apt/lists/* @@ -191,7 +198,7 @@ ARG GITHUB_CLI_VERSION="1.13.1" RUN BASE="https://github.com/cli/cli/releases/download" \ && echo "${BASE}/v${GITHUB_CLI_VERSION}/gh_${GITHUB_CLI_VERSION}_linux_${TARGETPLATFORM#linux/}.deb" \ && curl -fSsL "${BASE}/v${GITHUB_CLI_VERSION}/gh_${GITHUB_CLI_VERSION}_linux_${TARGETPLATFORM#linux/}.deb" \ - -o /tmp/gh.deb \ + -o /tmp/gh.deb \ && apt-get update \ && apt-get -y install /tmp/gh.deb \ && rm /tmp/gh.deb \ @@ -217,8 +224,12 @@ ARG NODE_MAJOR_VERSION="16" COPY superchain/gpg/nodesource.asc /tmp/nodesource.asc COPY superchain/gpg/yarn.asc /tmp/yarn.asc RUN apt-key add /tmp/nodesource.asc && rm /tmp/nodesource.asc \ - && echo "deb https://deb.nodesource.com/node_${NODE_MAJOR_VERSION}.x buster main" \ - > /etc/apt/sources.list.d/nodesource.list \ + && echo "deb https://deb.nodesource.com/node_${NODE_MAJOR_VERSION}.x ${DEBIAN_VERSION} main" \ + > /etc/apt/sources.list.d/nodesource.list \ + # Reduce priority of the "standard" nodejs package, so that the one from nodesource is always preferred... + && echo "Package: nodejs" > /etc/apt/preferences.d/nodejs \ + && echo 'Pin: origin "deb.debian.org"' >> /etc/apt/preferences.d/nodejs \ + && echo "Pin-Priority: 50" >> /etc/apt/preferences.d/nodejs \ && apt-key add /tmp/yarn.asc && rm /tmp/yarn.asc \ && echo "deb https://dl.yarnpkg.com/debian stable main" > /etc/apt/sources.list.d/yarnpkg.list \ && apt-get update \ @@ -228,9 +239,9 @@ RUN apt-key add /tmp/nodesource.asc && rm /tmp/nodesource.asc # Install Amazon SSM agent (allows debugging of builds via `codebuild-breakpoint`, https://go.aws/3TVW7vL) RUN apt-get update \ - && apt-get -y install curl \ - && curl -fSsL "https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_${TARGETPLATFORM#linux/}/amazon-ssm-agent.deb" \ - -o /tmp/amazon-ssm-agent.deb \ + && apt-get -y install --no-install-recommends curl \ + && curl -fSsL "https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_${TARGETPLATFORM#linux/}/amazon-ssm-agent.deb"\ + -o /tmp/amazon-ssm-agent.deb \ && dpkg -i /tmp/amazon-ssm-agent.deb \ && systemctl enable amazon-ssm-agent \ && rm -rf /var/lib/apt/lists/* diff --git a/superchain/README.md b/superchain/README.md index c8c9bcfafe..5bd89c07c9 100644 --- a/superchain/README.md +++ b/superchain/README.md @@ -1,6 +1,6 @@ # Superchain -A [`debian:10-slim`][debian]-based Docker image bundling all the SDKs and tools +A [`debian`][debian]-based Docker image bundling all the SDKs and tools required in order to package [jsii] projects in all supported languages. [debian]: https://gallery.ecr.aws/debian/debian @@ -28,12 +28,13 @@ public.ecr.aws/jsii/superchain:-(-node)(-nightly) - `` is the major line of the jsii toolchain - The only supported value is `1` -- `` is the base image tag (e.g: `buster-slim`) +- `` is the base image tag (e.g: `buster-slim`, `bullseye-slim`, `bookworm-slim`) - The only supported value is `buster-slim` - `` is the major version of node contained in the image - `14` corresponds to node 14.x, this is the default - `16` corresponds to node 16.x - `18` corresponds to node 18.x + - `20` corresponds to node 20.x - `-nightly` images are released from the `HEAD` of the [`aws/jsii`][jsii] repository and should typically not be used for production workloads @@ -70,6 +71,7 @@ We build multiple versions of this image, for different versions of Node. They a * `public.ecr.aws/jsii/superchain:1-buster-slim-node14(-nightly)` * `public.ecr.aws/jsii/superchain:1-buster-slim-node16(-nightly)` * `public.ecr.aws/jsii/superchain:1-buster-slim-node18(-nightly)` +* `public.ecr.aws/jsii/superchain:1-buster-slim-node20(-nightly)` If you are building this image from source, you can control the Node version with the `NODE_MAJOR_VERSION` build argument: diff --git a/superchain/build-local.sh b/superchain/build-local.sh index a933324dd2..ef7fceac3d 100755 --- a/superchain/build-local.sh +++ b/superchain/build-local.sh @@ -19,14 +19,20 @@ else COMMIT_ID=${COMMIT_ID}+local fi +PLATFORM=$(uname -m) +if [[ "${PLATFORM}" == "x86_64" ]]; then + PLATFORM="amd64" +fi + # Now on to building the image -docker build \ +${DOCKER:-docker} build \ --target superchain \ - --build-arg BUILDPLATFORM=linux/amd64 \ - --build-arg TARGETPLATFORM=linux/amd64 \ + --build-arg BUILDPLATFORM=linux/${PLATFORM} \ + --build-arg TARGETPLATFORM=linux/${PLATFORM} \ + --build-arg DEBIAN_VERSION=${DEBIAN_VERSION:-bookworm} \ --build-arg BUILD_TIMESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg REGISTRY="docker.io/library" \ --build-arg COMMIT_ID=${COMMIT_ID} \ - -t "public.ecr.aws/jsii/superchain:local" \ + -t "public.ecr.aws/jsii/superchain:1-${DEBIAN_VERSION:-bookworm}-local" \ -f ${PWD}/Dockerfile \ ..