VSHA-536 Support virtio (#56)

Change drive inclusion behavior from keying off of the `TRANSPORT` value of `lsblk` to the `SUBSYSTEM`. The `TRANSPORT` value
makes it easy to see which bus is being utilized by a disk, however this value is NULL (empty) for `virtio` devices.

In order to include `virtio` devices, such as devices used by Google Cloud, we can key off of the `SUBSYSTEM` value instead. The `SUBSYSTEM` value
provides similar information, allowing us to continue choosing `sas`, `sata`, `nvme`, and `scsi` while excluding `usb` devices.

Some refactoring is included in this change to fix variables that were (in some cases) undefined.

Fixes `metal_resolve_disk`, which was returning disks even if they failed the conditional check.

Add `metal-log.sh` This hook will copy any and all current and future log files made by
metal in `/tmp`, presuming any new logs start with `^metal` and end with
`.log`.

90metalmdsquash/metal-lib.sh

@@ -69,6 +69,13 @@ fi
 }
 _load_dracut_dep
 
+##############################################################################
+# constant: METAL_DONE_FILE_PAVED
+#
+# Log directory.
+export METAL_LOG_DIR='/var/log/metal'
+mkdir -p $METAL_LOG_DIR
+
 ##############################################################################
 # constant: METAL_DONE_FILE_PAVED
 #
@@ -79,12 +86,21 @@ _load_dracut_dep
 export METAL_DONE_FILE_PAVED='/tmp/metalpave.done'
 
 ##############################################################################
-# constant: metal_transports
+# constant: metal_subsystems
 #
-# PIPE-DELIMITED-LIST of Transports to acknowledge from `lsblk` queries; these transports are 
-# exclusively cleaned and partitioned, all others on the node are left alone.
+# PIPE-DELIMITED-LIST of SUBSYSTEMS to acknowledge from `lsblk` queries; anything listed here is in
+# the cross-hairs for wiping and formatting.
+# NOTE: To find values for this, run `lsblk -b -l -d -o SIZE,NAME,TYPE,SUBSYSTEMS`
 # MAINTAINER NOTE: DO NOT ADD USB or ANY REMOVABLE MEDIA TRANSPORT in order to mitigate accidents.
-export metal_transports="sata|nvme|sas"
+export metal_subsystems='scsi|nvme'
+
+##############################################################################
+# constant: metal_subsystems_ignore
+#
+# PIPE-DELIMITED-LIST of Transports to acknowledge from `lsblk` queries; these subsystems are
+# excluded from any operations performed by this dracut module.
+# NOTE: To find values for this, run `lsblk -b -l -d -o SIZE,NAME,TYPE,SUBSYSTEMS`
+export metal_subsystems_ignore='usb'
 
 ##############################################################################
 # costant: metal_fstab
@@ -103,7 +119,7 @@ export metal_fsopts_xfs=noatime,largeio,inode64,swalloc,allocsize=131072k
 #
 # Define the size that is considered to fit the "small" disk form factor. These
 # usually serve critical functions.
-export metal_disk_small=524288000000
+export metal_disk_small=375809638400
 
 ##############################################################################
 # constant: metal_disk_large
@@ -175,68 +191,58 @@ metal_die() {
     fi
 }
 
-
 ##############################################################################
-# function: metal_resolve_disk
+# function: metal_scand
 #
-# Function returns a space delemited list of tuples, each tuple contains the
-# size (in bytes) of a disk, and the disk handle itself. This output is
-# compatible with metal_resolve_disk.
+# Returns a sorted, space delimited list of disks. Each element in the list is
+# a tuple representing a disk; the size of the disk (in bytes), and
+# device-mapper name.
 #
 # usage:
 #
-#   Return disks except, ignoring the first two used by the OS:
-# 
-#       metal_scand $((metal_disks + 1))
-# 
-#   Return the OS disks:
+#     metal_scand
 #
-#      md_disks=();for disk in seq 1 $metal_disks; do md_disk=$(metal_scand $disk | cut -d ' ' -f1) ; echo $md_disk; md_disks+=( $md_disk ); done; echo ${md_disks[@]}
+# output:
+#
+#     10737418240,sdd 549755813888,sda 549755813888,sdb 1099511627776,sdc
 #
 metal_scand() {
-    local disk_offset=${1:-$metal_disks}
-    local disks
-    disks="$(lsblk -b -l -d -o SIZE,NAME,TYPE,TRAN |\
-        grep -E '('"$metal_transports"')' |\
+    echo -n "$(lsblk -b -l -d -o SIZE,NAME,TYPE,SUBSYSTEMS |\
+        grep -E '('"$metal_subsystems"')' |\
+        grep -v -E '('"$metal_subsystems_ignore"')' |\
         sort -h |\
         grep -vE 'p[0-9]+$' |\
-        awk '{print $1 "," $2}' |\
-        tail -n +${disk_offset} |\
+        awk '{print ($1 > '$metal_ignore_threshold') ? $1 "," $2 : ""}' |\
         tr '\n' ' ' |\
         sed 's/ *$//')"
-    echo $disks
 }
 
 ##############################################################################
 # function: metal_resolve_disk
 #
-# Sorts a list of disks, returning the first disk that's larger than the 
-# given constraint.
-#
-# The output of this lsblk command is ideal for this function:
+# Given a disk tuple from metal_scand and a minimum size, print the disk if it's
+# larger than the given size otherwise print nothing.
+# Also verified whether the disk has children or not, if it does then it's not
+# eligible. Since all disks are wiped to start with, if a disk has children when
+# this function would be called then it's already spoken for.
 #
-#   lsblk -b -l -o SIZE,NAME,TYPE,TRAN | grep -E '(sata|nvme|sas)' | sort -h | awk '{print $1 "," $2}' 
+# This is useful for iterating through a list of devices and ignoring ones that
+# are insufficient.
 #
 # usage:
 #
-#   metal_resolve_disk "size,name [size,name]" floor/minimum_size
+#   metal_resolve_disk size,name floor/minimum_size
 #
-# example(s):
-#
-#   metal_resolve_disk "480103981056,sdc 1920383410176,sdb" 1048576000000
 metal_resolve_disk() {
-    local disks=$1
+    local disk=$1
     local minimum_size=$(echo $2 | sed 's/,.*//')
-    local found=0
-    for disk in $disks; do
-        name="$(echo $disk | sed 's/,/ /g' | awk '{print $2}')"
-        size="$(echo $disk | sed 's/,/ /g' | awk '{print $1}')"
-        if [ "${size}" -gt $minimum_size ]; then
-            found=1
-        fi 
-    done
-    printf $name
-    [ $found = 1 ] && return 0 || return 1
+    name="$(echo $disk | sed 's/,/ /g' | awk '{print $2}')"
+    size="$(echo $disk | sed 's/,/ /g' | awk '{print $1}')"
+    if ! lsblk --fs --json "/dev/${name}" | grep -q children ; then
+        if [ "${size}" -gt "${minimum_size}" ]; then
+            echo -n "$name"
+        fi
+    fi
 }
 
 ##############################################################################
@@ -257,12 +263,10 @@ metal_paved() {
         case "$rc" in
             1)
                 # 1 indicates the pave function ran and the disks were wiped.
-                echo >&2 'Disks have been wiped.'
                 return 0
                 ;;
             0)
                 # 0 indicates the pave function was cleanly bypassed.
-                echo >&2 'Wipe was skipped.'
                 return 0
                 ;;
             *)

90metalmdsquash/metal-log.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# MIT License
+#
+# (C) Copyright 2023 Hewlett Packard Enterprise Development LP
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# metal-log.sh
+[ "${metal_debug:-0}" = 0 ] || set -x
+
+command -v disks_exist > /dev/null 2>&1 || . /lib/metal-lib.sh
+
+mkdir -p "/sysroot${METAL_LOG_DIR}/bootstrap"
+cp -p "${METAL_LOG_DIR}/"* "/sysroot${METAL_LOG_DIR}/bootstrap/" 2>/dev/null

90metalmdsquash/metal-md-disks.sh

@@ -47,22 +47,31 @@ pave
 
 # At this point this module is required; a disk must be created or the system has nothing to boot.
 # Die if no viable disks are found; otherwise continue to disk creation functions.
-if [ ! -f /tmp/metalsqfsdisk.done ]; then
+if [ ! -f /tmp/metalsqfsdisk.done ] && [ "${metal_nowipe}" -eq 0 ]; then
     md_disks=()
-    for disk in $(seq 1 $metal_disks); do
-        md_disk=$(metal_resolve_disk $(metal_scand $disk) $metal_disk_small)
-        md_disks+=( $md_disk )
+    disks="$(metal_scand)"
+    IFS=" " read -r -a pool <<< "$disks"
+    for disk in "${pool[@]}"; do
+        if [ "${#md_disks[@]}" -eq "${metal_disks}" ]; then
+            break
+        fi
+        md_disk=$(metal_resolve_disk "$disk" "$metal_disk_small")
+        if [ -n "${md_disk}" ]; then
+            md_disks+=("$md_disk")
+        fi
     done
-    if [ ${#md_disks[@]} = 0 ]; then
-        metal_die "No disks were found for the OS that were [$metal_disk_small] (in bytes) or smaller!"
+
+    if [ "${#md_disks[@]}" -lt "$metal_disks" ]; then
+        metal_die "No disks were found for the OS that were [$metal_disk_small] (in bytes) or larger, all were too small or had filesystems present!"
         exit 1
     else
         echo >&2 "Found the following disks for the main RAID array (qty. [$metal_disks]): [${md_disks[*]}]"
     fi
 fi
 
 # Create disks.
-[ ! -f /tmp/metalsqfsdisk.done ] && make_raid_store
+[ ! -f /tmp/metalsqfsdisk.done ] && make_raid_store "${md_disks[@]}"
+[ ! -f /tmp/metalovaldisk.done ] && make_raid_overlay "${md_disks[@]}"
 [ ! -f /tmp/metalovalimg.done ] && add_overlayfs
 [ ! -f /tmp/metalsqfsimg.done ] && add_sqfs