build_sdk_container_image

#!/bin/bash
#
# Copyright (c) 2021 The Flatcar Maintainers.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This script will generate an SDK container image from an SDK tarball
#  (which in turn is generated by bootstrap_sdk[_container]).
#
# It uses a number of intermediate build steps:
#   1. Import the SDK tarball into a "tarball" container image.
#   2. Build a "plain" SDK container image which creates the "sdk" user
#      and runs "update_chroot" to initialise the x86 and aarch64 SDK cross env.
#      This step uses sdk_lib/Dockerfile.sdk-import.
#   3. Using the "plain" SDK image, start a temporary "toolchain" container
#      to build toolchain binary packages.
#   4. Inheriting from the "plain" SDK image and using the toolchain packages,
#      build a "full" SDK container image, with board support for both amd64-usr and arm64-usr.
#      A temporary HTTP server on the Docker interface IP is spun up in this step
#      to serve the toolchain binpkgs to the build container.
#      This step uses sdk_lib/Dockerfile.sdk-build.
#   5. In a last step, all unnecessary binaries are removed from the "full" image and 3
#      output SDK images are produced:
#      - "all", with both amd64 and arm64 target support, and
#      - "amd64" and "arm64, with only amd64 or arm64 target support respectively.
#      This step uses sdk_lib/Dockerfile.lean-arch.

set -eu

cd $(dirname "$0")
source sdk_lib/sdk_container_common.sh

arch="amd64"
official="0"
tarball=""
os_version=""

keep="false"
cleanup=""

usage() {
    echo "  $0 - Create SDK container image from an SDK tarball"
    echo "       This script will set up a new SDK container from a tarball."
    echo "       The resulting container comes in 3 flavours:"
    echo "        1.    flatcar-sdk-all - includes both ARM64 and AMD64 support"
    echo "        2.+3. flatcar-sdk-(amd64|arm64) - only includes support for one target."
    echo "  Usage:"
    echo "  $0 <tarball>] [-k] [-v <os-version>] [-x <script>]"
    echo
    echo "      <tarball>    - Local tarball to build SDK from."
    echo "                     The tarball must follow the format"
    echo "                     flatcar-sdk-(amd64|arm64)-<version>.tar.bz2."
    echo "      -v <version> - Use custom OS version (defaults to tarball's SDK version)."
    echo "      -k           - Keep intermediate build containers (sdk-import / sdk-tarball)."
    echo "                     (Useful for sdk container build script development.)"
    echo "      -x <script>  - For each resource generated during build (container etc.)"
    echo "                     add a cleanup line to <script> which, when run, will free"
    echo "                     the resource. Useful for CI."
    echo "      -h           - Print this help."
    echo
}

# --

while [ 0 -lt $# ] ; do
    case "$1" in
    -h) usage; exit 0;;
    -k) keep="true";      shift;;
    -v) os_version="$2";  shift; shift;;
    -x) cleanup="$2";     shift; shift;;
    *)  if [ -z "$tarball" ] ;  then
            tarball="$1"; shift
        else
            echo "ERROR: spurious positional parameter '$@'."
            usage
            exit 1
        fi;;
    esac
done

if [ -z "$tarball" -o ! -s "$tarball" ] ; then
    echo "ERROR: missing / invalid SDK tarball argument"
    exit 1
fi
# --

# Grok version / arch from tarball name, set official if version is a release version
version="$(echo "$tarball" | sed -n 's/.*flatcar-sdk-\(arm64\|amd64\)-\(.\+\)\.tar\.bz2/\2/p')"
arch="$(echo "$tarball" | sed -n 's/.*flatcar-sdk-\(arm64\|amd64\)-.*\.tar\.bz2/\1/p')"
if [ -z "$version" -o -z "$arch" ]; then
    echo "ERROR: Unable to determine version / arch from '$tarball'"
    exit 1
fi

if [ -z "${os_version}" ] ; then
    os_version="${version}"
fi

if is_official "$version" && [ "${version}" = "${os_version}" ] ; then
    official="1"
else
    official="0"
fi

# --

# import tarball
#
yell "\n######\n###### Building SDK container for version $version from '$tarball'"
create_versionfile "$version" "${os_version}"

docker_vernum="$(vernum_to_docker_image_version "${version}")"
import_tarball="flatcar-sdk-tarball:${docker_vernum}"
image_present="$($docker image ls "$import_tarball" --format '{{.Repository}}:{{.Tag}}')"

if [ "${image_present}" = "${import_tarball}" ] ; then
    yell "Using existing SDK tarball image '${import_tarball}'"
else
    yell "Importing SDK tarball"
    if [ -n "$cleanup" ] ; then
        echo "$docker image rm -f '${import_tarball}'" >> "$cleanup"
    fi
    $docker import "${tarball}" "${import_tarball}"
fi

# --

# build plain SDK container w/o board support
#
import_image="flatcar-sdk-import:${docker_vernum}"
image_present="$($docker image ls "${import_image}" --format '{{.Repository}}:{{.Tag}}')"

if [ "$image_present" = "${import_image}" ] ; then
    yell "Using existing SDK import image '${import_image}'"
else
    yell "Building plain SDK import image"
    if [ -n "$cleanup" ] ; then
        echo "$docker image rm -f '${import_image}'" >> "$cleanup"
    fi
    $docker build -t "$import_image" \
                 --build-arg VERSION="${docker_vernum}" \
                 -f sdk_lib/Dockerfile.sdk-import \
                 .
fi

# --

# build full SDK container w/ board support.
#  This uses the SDK import container to first build toolchain binpkgs.
#  Then, the import container and toolchain packages are used
#  to build a full SDK container w/ amd64 and arm64 board support.
#
sdk_build_image="flatcar-sdk-build:${docker_vernum}"
image_present="$($docker image ls "${sdk_build_image}" --format '{{.Repository}}:{{.Tag}}')"
if [ "$image_present" = "${sdk_build_image}"  ] ; then
    yell "Using existing SDK build image '${sdk_build_image}'"
else
    # --- Toolchains build ---
    yell "Building toolchains in a temporary container."
    # We need to use run_sdk_container instead of building from a Dockerfile
    # since toolchains build uses catalyst which requires privileged access.
    tarball_copied=""
    if [ "$(basename "${tarball}")" != "${tarball}" ] ; then
        cp --reflink=auto "${tarball}" ./
        tarball="$(basename "${tarball}")"
        tarball_copied="${tarball}"
    fi

    toolchains_container="flatcar-sdk-toolchains-build-${docker_vernum}"
    if [ -n "$cleanup" ] ; then
        echo "$docker container rm -f '${toolchains_container}'" >> "$cleanup"
    fi
    ./run_sdk_container -C "${import_image}" -n "${toolchains_container}" \
            sudo ./build_toolchains --seed_tarball="./${tarball}"

    # remove sdk tarball from scripts root so it's not part of the SDK container build context
    if [ -f "${tarball_copied}" ] ; then
        rm "${tarball_copied}"
    fi

    $docker container rm -f "${toolchains_container}"

    docker_interface="docker0"
    if "${is_podman}"; then
      # Make a dummy run without "--net host" here for the interface to be created
      $docker run --rm alpine
      docker_interface="cni-podman0"
    fi
    host_ip="$(ip addr show "${docker_interface}" | grep -Po 'inet \K[\d.]+')"
    binhost_port="$((1000 + (RANDOM % 55000) ))"
    binhost="${host_ip}:${binhost_port}"
    binhost_container="${toolchains_container}-binhost-${binhost_port}"
    yell "Building SDK container + board support, toolchain packages served at http://${binhost} by ${binhost_container}"

    # Spin up temporary toolchains package binhost
    if [ -n "$cleanup" ] ; then
        echo "$docker container rm -f '${binhost_container}'" >> "$cleanup"
    fi
    $docker run --rm -d -p "${binhost}":80 \
        --name ${binhost_container} \
        -v "$(pwd)/__build__/images/catalyst/packages/coreos-toolchains/target":/usr/share/caddy \
        docker.io/library/caddy caddy file-server \
        --root /usr/share/caddy --browse

    # --- Full SDK container build ---

    yell "Initialising the SDK container and building board packages"
    if [ -n "$cleanup" ] ; then
        echo "$docker image rm -f '${sdk_build_image}'" >> "$cleanup"
    fi
    $docker build -t "${sdk_build_image}" \
                 --build-arg VERSION="${docker_vernum}" \
                 --build-arg BINHOST="http://${binhost}" \
                 --build-arg OFFICIAL="${official}" \
                 -f sdk_lib/Dockerfile.sdk-build \
                 .

    $docker stop "${binhost_container}"
fi

# --

# Derive "lean" SDK containers from full build. Main purpose
#  of this step is to remove "white-outs", i.e. files which have been
#  deleted in the full image but are still present in an intermediate layer.
#
for a in all arm64 amd64; do
    yell "Creating '$a' arch SDK image"
    rmarch=""; rmcross=""
    case $a in
        arm64) rmarch="amd64-usr"; rmcross="x86_64-cros-linux-gnu";;
        amd64) rmarch="arm64-usr"; rmcross="aarch64-cros-linux-gnu";;
    esac
    $docker build -t "$sdk_container_common_registry/flatcar-sdk-${a}:${docker_vernum}" \
                 --build-arg VERSION="${docker_vernum}" \
                 --build-arg RMARCH="${rmarch}" \
                 --build-arg RMCROSS="${rmcross}" \
                 -f sdk_lib/Dockerfile.lean-arch \
                 .
done

# --

# Cleanup
#
if ! $keep; then
    yell "Cleaning up intermediate containers"
    $docker rmi flatcar-sdk-build:"${docker_vernum}"
    $docker rmi flatcar-sdk-import:"${docker_vernum}"
    $docker rmi flatcar-sdk-tarball:"${docker_vernum}"
fi