Build on top of Andrew's change to bootstrap without docker

Address Snaipe's comments:
  - Elaborate on the description for bootstrap image
  - Compare checksums wrt to a file.
  - Combine bootstrap image generation sub images into a single one
    to avoid caching the same thing multiple times, the steps are fast.
  - Create mountpoints in the bootstrap image

Additional changes:
  - CHECKSUM file, dnf/yum repo configuration and helper scripts are now
    checked in to the repo as opposed to autogenerated in the build step to
    make maintenance/upgrades easier and the barney.yaml a bit more
    readable.
  - Instead of downloading the tarball from the internet, download it
    from a stashed copy in a local artifactory repo.

barney.yaml

@@ -6,149 +6,104 @@ generators:
 images:
 
   internal/bootstrap/iso-extraction-floor:
+    entry:
+      mutables:
+        - /var/work
     units:
       - image: barney.ci/alpine%pkg/alpine-base
+      - image: barney.ci/alpine%pkg/gettext
       - image: barney.ci/alpine%pkg/wget
       - image: barney.ci/alpine%network
       - image: barney.ci/alpine%apk-finalizers
+      - build: |
+          mkdir -p /dest/var/work
 
-  internal/bootstrap/base.tar.xz:
-    description: |
-      Downloading a recent-ish centos container base from the upstream
-      centos registry. Note that we cache this step separately for quick
-      development.
+  internal/bootstrap/src-deps:
     no-create-mountpoints: true
     units:
-      - floor: .%internal/bootstrap/iso-extraction-floor
-        sources: []
-        build: |
-          arch=$(uname -m)
-          case "$arch" in
-            x86_64)
-              arch=x86_64
-              cksum=63b7ddb444b23a07cb851398c338595e410fb3fac2dd72061d0292c653e5afe6
-              ;;
-            i?86)
-              arch=x86_64
-              cksum=63b7ddb444b23a07cb851398c338595e410fb3fac2dd72061d0292c653e5afe6
-              ;;
-            aarch64)
-              arch=arm64
-              cksum=312a833dfe646ce5b41f362cae577df9797955b85ced96173be8e88e5ebd5990
-              ;;
-            *)
-              >&2 echo unsupported architecture "$arch"
-              exit 1
-              ;;
-          esac
-
-          cd /dest
-          wget https://cloud.centos.org/centos/9-stream/${arch}/images/CentOS-Stream-Container-Base-9-20230501.0.${arch}.tar.xz \
-            --output-document base.tar.xz
-
-          echo "$cksum  base.tar.xz" | sha256sum -c
-
-  internal/bootstrap/extract/1:
-    no-create-mountpoints: true
-    units:
-      - floor: .%internal/bootstrap/iso-extraction-floor
-        sources: []
-        mappings:
-          /src/base: .%internal/bootstrap/base.tar.xz
-        build: |
-          tar --strip-components=1 -xf /src/base/base.tar.xz -C /dest
+      - build: |
+          cp -a bootstrap/* /dest/
+          cp -a pki/rpmkeys/alma9-b86b3716-gpg-pubkey.pem /dest/
+          cp -a pki/rpmkeys/epel9-3228467c-gpg-pubkey.pem /dest/
+          rm -f /dest/README*
 
-  internal/bootstrap/extract/2:
+  internal/bootstrap:
     description: |
-      Extract our bootstrapping environment and remove any pre-configured
-      yum repos. This bootstrapping environment will be centos 9 stream,
-      but because we will install el9 repos under /etc/yum.repos.d, the
-      environments that we boostrap will be el9.
-    no-create-mountpoints: true
-    units:
-      - floor: .%internal/bootstrap/iso-extraction-floor
-        sources: []
-        mappings:
-          /src/layer: .%internal/bootstrap/extract/1
-        build: |
-          tar -xf /src/layer/layer.tar -C /dest
-          rm /dest/etc/yum.repos.d/*
+      Minimal bootstrapping environment used to bootstrap almalinux.
 
-  internal/bootstrap/install-rpms:
-    description: |
-      The install-rpms command accepts a list of packages and installs them
-      (along with dependencies) into /dest. This is useful for creating new
-      chroot environments with an arbitrary set of yum repos.
-    no-create-mountpoints: true
-    units:
-      - sources: []
-        build: |
-          mkdir -p /dest/usr/bin
-          chmod 555 /dest/usr/bin
-          echo '#!/bin/sh
-          dnf --assumeyes --installroot=/dest --noplugins \
-              --config=/etc/dnf/dnf.conf \
-              --setopt=cachedir=/var/cache/microdnf \
-              --setopt=reposdir=/etc/yum.repos.d \
-              --setopt=varsdir=/etc/dnf --releasever=9.1 install "$@"
-          ' > /dest/usr/bin/install-rpms
-          chmod 755 /dest/usr/bin/install-rpms
-
-  internal/bootstrap/repos:
-    description: |
-      Install yum repos associated with el9 under /etc/yum.repos.d. Intended to
-      be used in combination with the install-rpms script.
-    no-create-mountpoints: true
-    units:
-      - sources: []
-        entry:
-          env:
-            DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory}
-        build: |
-          export DNF_ARCH="$(arch)"
-          export DNF_DISTRO_REPO="alma-vault/9.1"
-          export DNF_EPEL9_REPO_VERSION="v20240127-1"
-          export DNF_EPEL9_REPO="eext-snapshots-local/epel9/${DNF_EPEL9_REPO_VERSION}/9/Everything"
-          mkdir -p /dest/etc/yum.repos.d
-          echo "[epel9-subset]
-          baseurl=${DNF_HOST}/${DNF_EPEL9_REPO}/${DNF_ARCH}/
-          enabled=1
-          gpgcheck=0
-          " > /dest/etc/yum.repos.d/eext-externaldeps.repo
-          echo "[BaseOS]
-          baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/BaseOS/${DNF_ARCH}/os/
-          gpgcheck=0
-          enabled=1
-          " > /dest/etc/yum.repos.d/BaseOS.repo
-          echo "[AppStream]
-          baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/AppStream/${DNF_ARCH}/os/
-          exclude=podman
-          gpgcheck=0
-          enabled=1
-          " > /dest/etc/yum.repos.d/AppStream.repo
-
-  internal/bootstrap/network:
+      This needs to be based off of an rpm-based distro,
+      we're using CentOS-stream as a suitable tarball and CHECKSUM file
+      is readily available on the mirror. This is seldom changed and
+      any change here will update all eext snapshots.
+
+      We wipe out the dnf/yum repo configuration in the tarball,
+      and setup our own curated set of frozen almalinux repos.
+
+      We also include an install-rpms script which is used to bootstrap
+      the almalinux image.
     entry:
       share-net: true
       mounts:
         - source: /etc/resolv.conf
           target: /etc/resolv.conf
           options: ro,bind
-
-  internal/bootstrap:
-    description: |
-      Minimal bootstrapping environment. Do not run builds in
-      this directly, but instead use it to create images that
-      contain a specific set of dependencies.
-    entry:
       mutables:
         - /var/cache
         - /var/lib/dnf
     units:
-      - image: .%internal/bootstrap/extract/2
-      - image: .%internal/bootstrap/repos
-      - image: .%internal/bootstrap/install-rpms
-      - image: .%internal/bootstrap/network
+      - floor: .%internal/bootstrap/iso-extraction-floor
+        entry:
+          env:
+            DNF_HOST: ${eext-dnf-host.url:-https://artifactory.infra.corp.arista.io/artifactory}
+            BOOTSTRAP_PATH: eext-sources/bootstrap/CentOS-Stream
+            BOOTSTRAP_FILE: CentOS-Stream-Container-Base-9-20240715.0
+            BOOTSTRAP_EXTENSION: tar.xz
+            DNF_DISTRO_REPO: alma-vault
+            DNF_DISTRO_REPO_VERSION: "9.3"
+            DNF_EPEL_REPO: eext-snapshots-local/epel9
+            DNF_EPEL_REPO_SNAPSHOT_VERSION: v20240127-1
+            DNF_EPEL_REPO_DISTRO_VERSION: "9"
+        sources: []
+        mappings:
+          /src/bootstrap: .%internal/bootstrap/src-deps
+        build: |
+          # URL of tarball with OS image
+          export ARCH=$(uname -m)
+          bootstrap_filename="${BOOTSTRAP_FILE}.${ARCH}.${BOOTSTRAP_EXTENSION}"
+          bootstrap_url="${DNF_HOST}/${BOOTSTRAP_PATH}/${bootstrap_filename}"
+
+          # Download the tarball into the mutable working dir
+          cd /var/work
+          wget ${bootstrap_url}
+
+          # Validate downloaded tarball
+          grep "${bootstrap_filename}" /src/bootstrap/CHECKSUM | sha256sum -wc
+
+          # Extract tarball and setup rootfs
+          # This is a nested tarball, the real rootfs is in layer.tar
+          # Extract the firt level tarball inside the extr subdirectory
+          # within the working directory and and then extract the
+          # second level layer.tar directly to /dest
+          mkdir extr
+          tar --strip-components=1 -C ./extr -xf ./${bootstrap_filename}
+          tar -xf ./extr/layer.tar -C /dest
+
+          # Now modify the extracted file system to remove unwanted
+          # stuff and add extra stuff.
+
+          # Remove any pre-configured yum repos
+          rm /dest/etc/yum.repos.d/*
+          # Setup install-rpms shell script
+          mkdir -p /dest/usr/bin
+          chmod 555 /dest/usr/bin
+          cp -a /src/bootstrap/install-rpms /dest/usr/bin/
+          chmod 0755 /dest/usr/bin/install-rpms
+          # Setup gpg keys
+          mkdir -p /dest/usr/share/distribution-gpg-keys
+          chmod 555 /dest/usr/share/distribution-gpg-keys
+          cp -a /src/bootstrap/*.pem /dest/usr/share/distribution-gpg-keys/
+          # Setup curated frozen dnf repo list for build reproducibility
+          envsubst < /src/bootstrap/eext-repos-build.repo.template > /dest/etc/yum.repos.d/eext-repos-build.repo
 
   base-image:
     units:

bootstrap/CHECKSUM

@@ -0,0 +1,2 @@
+c2a4f505ffa543fa46f9a077d201d54d8d94f431079b5e776c15cb200c03bb73 CentOS-Stream-Container-Base-9-20240715.0.x86_64.tar.xz
+af064c53839e2b54c7a42f7c6d1f4c4ed597e4ba6981ea90a1e6914278dfe855 CentOS-Stream-Container-Base-9-20240715.0.aarch64.tar.xz

bootstrap/README.md

@@ -0,0 +1,13 @@
+# Uploading Bootstrap Tarball
+
+1. Download a CentOS-Stream-Container-Base tarball with a timestamp in its name, like `20230704`, from:
+   - [https://cloud.centos.org/centos/9-stream/x86_64/images/](https://cloud.centos.org/centos/9-stream/x86_64/images/)
+   - [https://cloud.centos.org/centos/9-stream/aarch64/images/](https://cloud.centos.org/centos/9-stream/aarch64/images/)
+2. Upload them to artifactory in the subpath `eext-sources/bootstrap/CentOS-Stream/`
+```
+  curl -H "Authorization: Bearer ${ARTIFACTORY_TOKEN}" -X PUT https://artifactory.infra.corp.arista.io/artifactory/eext-sources/bootstrap/CentOS-Stream/ -T <TARBALL_PATH>
+```
+3. Update the `CHECKSUM` file in the local repo for the new entries from the `CHECKSUM` files:
+   - [https://cloud.centos.org/centos/9-stream/x86_64/images/CHECKSUM](https://cloud.centos.org/centos/9-stream/x86_64/images/CHECKSUM)
+   - [https://cloud.centos.org/centos/9-stream/aarch64/images/CHECKSUM](https://cloud.centos.org/centos/9-stream/aarch64/images/CHECKSUM)
+4. Update the `EEXT_BOOTSTRAP_VERSION` environment variable in `barney.yaml`.

bootstrap/eext-repos-build.repo.template

@@ -0,0 +1,16 @@
+[BaseOS]
+baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/BaseOS/${ARCH}/os/
+enabled=1
+gpgkey=file:///usr/share/distribution-gpg-keys/alma9-b86b3716-gpg-pubkey.pem
+
+[AppStream]
+baseurl=${DNF_HOST}/${DNF_DISTRO_REPO}/${DNF_DISTRO_REPO_VERSION}/AppStream/${ARCH}/os/
+exclude=podman
+enabled=1
+gpgkey=file:///usr/share/distribution-gpg-keys/alma9-b86b3716-gpg-pubkey.pem
+
+[epel9-snapshot]
+baseurl=${DNF_HOST}/${DNF_EPEL_REPO}/${DNF_EPEL_REPO_SNAPSHOT_VERSION}/${DNF_EPEL_REPO_DISTRO_VERSION}/Everything/${ARCH}/
+enabled=1
+gpgkey=file:///usr/share/distribution-gpg-keys/epel9-3228467c-gpg-pubkey.pem
+

bootstrap/install-rpms

@@ -0,0 +1,7 @@
+#!/bin/sh
+dnf --assumeyes --installroot=/dest --noplugins \
+    --config=/etc/dnf/dnf.conf \
+    --setopt=cachedir=/var/cache/dnf \
+    --setopt=reposdir=/etc/yum.repos.d \
+    --setopt=varsdir=/etc/dnf \
+    install "$@"