osbuild: Add Live ISO/PXE build support

.cci.jenkinsfile

@@ -29,14 +29,8 @@ pod(image: imageName + ":latest", kvm: true, cpu: "${cpuCount}", memory: "${memo
     // Run stage Kola QEMU (basic-qemu-scenarios, upgrade and self tests)
     kola(cosaDir: "/srv", addExtTests: ["${env.WORKSPACE}/ci/run-kola-self-tests"])
 
-    stage("Build Metal") {
-        utils.cosaCmd(cosaDir: "/srv", args: "osbuild metal metal4k")
-    }
-
-    stage("Build Live Images") {
-        // Explicitly test re-importing the ostree repo
-        shwrap("cd /srv && rm tmp/repo -rf")
-        utils.cosaCmd(cosaDir: "/srv", args: "buildextend-live --fast")
+    stage("Build Metal/Live Artifacts") {
+        utils.cosaCmd(cosaDir: "/srv", args: "osbuild metal metal4k live")
     }
 
     kolaTestIso(cosaDir: "/srv")

build.sh

@@ -177,6 +177,11 @@ patch_osbuild() {
         /usr/lib/coreos-assembler/0002-parsing-treat-locations-without-scheme-as-belonging-.patch \
         /usr/lib/coreos-assembler/0003-org.osbuild.selinux-support-operating-on-mounts.patch      \
         /usr/lib/coreos-assembler/0004-org.osbuild.selinux-support-for-specifying-where-fil.patch \
+        /usr/lib/coreos-assembler/0001-osbuild-remoteloop-add-more-loop-device-options.patch      \
+        /usr/lib/coreos-assembler/0002-osbuild-loop-make-the-loop-device-if-missing.patch         \
+        /usr/lib/coreos-assembler/0003-util-osrelease.py-improve-whitespace-and-quote-strip.patch \
+        /usr/lib/coreos-assembler/0004-util-chroot-Add-support-for-custom-directory-bind-mo.patch \
+        /usr/lib/coreos-assembler/0005-stages-add-coreos.live-artifacts.mono-stage.patch          \
             | patch -d /usr/lib/osbuild -p1
 
     # And then move the files back; supermin appliance creation will need it back

mantle/cmd/kola/testiso.go

@@ -819,7 +819,7 @@ func testPXE(ctx context.Context, inst platform.Install, outdir string) (time.Du
 	liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable)
 
 	if isOffline {
-		contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.BuildID, kola.CosaBuild.Meta.OstreeCommit)
+		contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion, kola.CosaBuild.Meta.OstreeCommit)
 		liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable)
 	}
 

mantle/platform/metal.go

@@ -668,6 +668,10 @@ func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgni
 	if err := cmd.Run(); err != nil {
 		return nil, errors.Wrapf(err, "copying iso")
 	}
+	// Make it writable so we can modify it
+	if err := os.Chmod(newIso, 0644); err != nil {
+		return nil, errors.Wrapf(err, "setting permissions on iso")
+	}
 	srcisopath = newIso
 
 	var metalimg string

mantle/platform/qemu.go

@@ -1500,6 +1500,10 @@ func (builder *QemuBuilder) setupIso() error {
 	if err := cpcmd.Run(); err != nil {
 		return errors.Wrapf(err, "copying iso")
 	}
+	// Make it writable so we can modify it
+	if err := os.Chmod(isoEmbeddedPath, 0644); err != nil {
+		return errors.Wrapf(err, "setting permissions on iso")
+	}
 	if builder.ConfigFile != "" {
 		if builder.configInjected {
 			panic("config already injected?")

src/0001-osbuild-remoteloop-add-more-loop-device-options.patch

@@ -0,0 +1,92 @@
+From 9edf8572ad4797033e7342c308ac617aa284f3ae Mon Sep 17 00:00:00 2001
+From: Dusty Mabe <dusty@dustymabe.com>
+Date: Fri, 22 Nov 2024 19:02:57 -0500
+Subject: [PATCH 1/5] osbuild/remoteloop: add more loop device options
+
+This adds lock, partscan, read_only, sector_size to _create_device()
+similar to make_loop() from devices/org.osbuild.loopback.
+---
+ osbuild/remoteloop.py | 42 +++++++++++++++++++++++++++++++++++++-----
+ 1 file changed, 37 insertions(+), 5 deletions(-)
+
+diff --git a/osbuild/remoteloop.py b/osbuild/remoteloop.py
+index 0544e0be..0fd2cfc0 100644
+--- a/osbuild/remoteloop.py
++++ b/osbuild/remoteloop.py
+@@ -41,8 +41,23 @@ class LoopServer(api.BaseAPI):
+         self.devs = []
+         self.ctl = loop.LoopControl()
+
+-    def _create_device(self, fd, dir_fd, offset=None, sizelimit=None):
+-        lo = self.ctl.loop_for_fd(fd, offset=offset, sizelimit=sizelimit, autoclear=True)
++    def _create_device(
++            self,
++            fd,
++            dir_fd,
++            offset=None,
++            sizelimit=None,
++            lock=False,
++            partscan=False,
++            read_only=False,
++            sector_size=512):
++        lo = self.ctl.loop_for_fd(fd, lock=lock,
++                                  offset=offset,
++                                  sizelimit=sizelimit,
++                                  blocksize=sector_size,
++                                  partscan=partscan,
++                                  read_only=read_only,
++                                  autoclear=True)
+         lo.mknod(dir_fd)
+         # Pin the Loop objects so they are only released when the LoopServer
+         # is destroyed.
+@@ -54,8 +69,12 @@ class LoopServer(api.BaseAPI):
+         dir_fd = fds[msg["dir_fd"]]
+         offset = msg.get("offset")
+         sizelimit = msg.get("sizelimit")
++        lock = msg.get("lock", False)
++        partscan = msg.get("partscan", False)
++        read_only = msg.get("read_only", False)
++        sector_size = msg.get("sector_size", 512)
+
+-        devname = self._create_device(fd, dir_fd, offset, sizelimit)
++        devname = self._create_device(fd, dir_fd, offset, sizelimit, lock, partscan, read_only, sector_size)
+         sock.send({"devname": devname})
+
+     def _cleanup(self):
+@@ -75,11 +94,20 @@ class LoopClient:
+             self.client.close()
+
+     @contextlib.contextmanager
+-    def device(self, filename, offset=None, sizelimit=None):
++    def device(
++            self,
++            filename,
++            offset=None,
++            sizelimit=None,
++            lock=False,
++            partscan=False,
++            read_only=False,
++            sector_size=512):
+         req = {}
+         fds = []
+
+-        fd = os.open(filename, os.O_RDWR)
++        flags = os.O_RDONLY if read_only else os.O_RDWR
++        fd = os.open(filename, flags)
+         dir_fd = os.open("/dev", os.O_DIRECTORY)
+
+         fds.append(fd)
+@@ -91,6 +119,10 @@ class LoopClient:
+             req["offset"] = offset
+         if sizelimit:
+             req["sizelimit"] = sizelimit
++        req["lock"] = lock
++        req["partscan"] = partscan
++        req["read_only"] = read_only
++        req["sector_size"] = sector_size
+
+         self.client.send(req, fds=fds)
+         os.close(dir_fd)
+-- 
+2.47.0
+

src/0002-osbuild-loop-make-the-loop-device-if-missing.patch

@@ -0,0 +1,34 @@
+From 4f09c4d4c1ad2026346a98e63f0e13155d0f0487 Mon Sep 17 00:00:00 2001
+From: Dusty Mabe <dusty@dustymabe.com>
+Date: Mon, 25 Nov 2024 16:29:10 -0500
+Subject: [PATCH 2/5] osbuild/loop: make the loop device if missing
+
+A few times during development I saw an error where the loop
+device wasn't getting created. Maybe it was some weird state
+issue with my system (i.e. loopback devices are global), or
+maybe not. Either way maybe it won't hurt to create it if
+it doesn't exist.
+---
+ osbuild/loop.py | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/osbuild/loop.py b/osbuild/loop.py
+index ec6d3619..b768af22 100644
+--- a/osbuild/loop.py
++++ b/osbuild/loop.py
+@@ -126,6 +126,12 @@ class Loop:
+             if not dir_fd:
+                 dir_fd = os.open("/dev", os.O_DIRECTORY)
+                 stack.callback(lambda: os.close(dir_fd))
++            # If the loopdev didn't show up for some reason let's
++            # create it manually
++            try:
++                os.stat(self.devname, dir_fd=dir_fd)
++            except FileNotFoundError:
++                self.mknod(dir_fd)
+             self.fd = os.open(self.devname, os.O_RDWR, dir_fd=dir_fd)
+
+         info = os.stat(self.fd)
+-- 
+2.47.0
+

src/0003-util-osrelease.py-improve-whitespace-and-quote-strip.patch

@@ -0,0 +1,32 @@
+From a835c409a19a9c72b5e669a70664ea960d873704 Mon Sep 17 00:00:00 2001
+From: Renata Ravanelli <rravanel@redhat.com>
+Date: Tue, 12 Nov 2024 15:12:52 -0300
+Subject: [PATCH 3/5] util/osrelease.py: improve whitespace and quote stripping
+
+- Enhanced the value stripping logic in osrelease parsing
+to handle leading and trailing spaces, newlines, tabs,
+and both single and double quotes.
+- This ensures cleaner and more accurate key-value assignments.
+
+Signed-off-by: Renata Ravanelli <rravanel@redhat.com>
+(cherry picked from commit 066f1ea89fbda6e886a5d88119586c0f09b0a234)
+---
+ osbuild/util/osrelease.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/osbuild/util/osrelease.py b/osbuild/util/osrelease.py
+index b8d56e73..a2b61d26 100644
+--- a/osbuild/util/osrelease.py
++++ b/osbuild/util/osrelease.py
+@@ -33,7 +33,7 @@ def parse_files(*paths):
+                 if line[0] == "#":
+                     continue
+                 key, value = line.split("=", 1)
+-                osrelease[key] = value.strip('"')
++                osrelease[key] = value.strip(" \n\t'\"")
+
+     return osrelease
+
+-- 
+2.47.0
+

src/0004-util-chroot-Add-support-for-custom-directory-bind-mo.patch

@@ -0,0 +1,60 @@
+From c367e6506ed9d4c674795ccc2da7a850c367398e Mon Sep 17 00:00:00 2001
+From: Renata Ravanelli <rravanel@redhat.com>
+Date: Thu, 31 Oct 2024 14:13:50 -0300
+Subject: [PATCH 4/5] util/chroot: Add support for custom directory bind mounts
+
+ - Add optional bind_mounts parameter to __init__ method;
+ - Enhanced methods to accept an optional `bind_mounts`.
+This allows for more flexible for configurations when setting
+up bind mounts.
+
+Signed-off-by: Renata Ravanelli <rravanel@redhat.com>
+(cherry picked from commit 9b5fbadee6b170455d62c57eb315e20d57173110)
+---
+ osbuild/util/chroot.py | 15 ++++++++++++++-
+ 1 file changed, 14 insertions(+), 1 deletion(-)
+
+diff --git a/osbuild/util/chroot.py b/osbuild/util/chroot.py
+index da14bf44..11245bbd 100644
+--- a/osbuild/util/chroot.py
++++ b/osbuild/util/chroot.py
+@@ -12,8 +12,9 @@ class Chroot:
+     This mounts /proc, /dev, and /sys.
+     """
+
+-    def __init__(self, root: str):
++    def __init__(self, root: str, bind_mounts=None):
+         self.root = root
++        self._bind_mounts = bind_mounts or []
+
+     def __enter__(self):
+         for d in ["/proc", "/dev", "/sys"]:
+@@ -33,6 +34,13 @@ class Chroot:
+                         "sysfs", f"{self.root}/sys"],
+                        check=True)
+
++        for d in self._bind_mounts:
++            target_path = os.path.join(self.root, d.lstrip("/"))
++            if not os.path.exists(target_path):
++                print(f"Making missing chroot directory: {d}")
++                os.makedirs(target_path)
++            subprocess.run(["mount", "--rbind", d, target_path], check=True)
++
+         return self
+
+     def __exit__(self, exc_type, exc_value, tracebk):
+@@ -43,6 +51,11 @@ class Chroot:
+         if failed_umounts:
+             print(f"Error unmounting paths from chroot: {failed_umounts}")
+
++        for d in self._bind_mounts[::-1]:
++            target_path = os.path.join(self.root, d.lstrip("/"))
++            if subprocess.run(["umount", "--lazy", target_path], check=False).returncode != 0:
++                print(f"Error unmounting paths from chroot: {d}")
++
+     def run(self, cmd, **kwargs):
+         cmd = ["chroot", self.root] + cmd
+         # pylint: disable=subprocess-run-check
+-- 
+2.47.0
+