Basic WIP cloud-init support

Signed-off-by: paulober <44974737+paulober@users.noreply.github.com>

README.md

@@ -205,6 +205,10 @@ The following environment variables are supported:
 
     If set, then instead of working through the numeric stages in order, this list will be followed. For example setting to `"stage0 stage1 mystage stage2"` will run the contents of `mystage` before stage2. Note that quotes are needed around the list. An absolute or relative path can be given for stages outside the pi-gen directory.
 
+  * `ENABLE_CLOUD_INIT` (Default: `1`)
+
+    If set to `1`, cloud-init and netplan will be installed and configured. This will allow you to configure your Raspberry Pi using cloud-init configuration files. The cloud-init configuration files should be placed in the bootfs or by editing the files in `stage2/04-cloud-init/files`. Cloud-init will be configured to read them on first boot.
+
 A simple example for building Raspberry Pi OS:
 
 ```bash

build.sh

@@ -243,6 +243,8 @@ export QUILT_NO_DIFF_INDEX=1
 export QUILT_NO_DIFF_TIMESTAMPS=1
 export QUILT_REFRESH_ARGS="-p ab"
 
+export ENABLE_CLOUD_INIT=${ENABLE_CLOUD_INIT:-1}
+
 # shellcheck source=scripts/common
 source "${SCRIPT_DIR}/common"
 # shellcheck source=scripts/dependencies_check

export-image/01-user-rename/01-run.sh

@@ -1,9 +1,21 @@
 #!/bin/bash -e
 
 if [[ "${DISABLE_FIRST_BOOT_USER_RENAME}" == "0" ]]; then
+	# with cloud-init enabled this will throw an error 
+	# when run more than once, as the service will be deleted
 	on_chroot <<- EOF
 		SUDO_USER="${FIRST_USER_NAME}" rename-user -f -s
 	EOF
+
+	# delete userconfig service as cloud-init will take care of launching it
+	rm -f "${ROOTFS_DIR}/lib/systemd/system/userconfig.service"
 else
 	rm -f "${ROOTFS_DIR}/etc/xdg/autostart/piwiz.desktop"
+
+	# if cloud-init enabled disable setup wizard launch completely
+	if [[ "${ENABLE_CLOUD_INIT}" == "1" ]]; then
+		on_chroot <<- EOF
+			touch /var/lib/userconf-pi/deactivate
+		EOF
+	fi
 fi

stage2/01-sys-tweaks/01-run.sh

@@ -1,13 +1,17 @@
 #!/bin/bash -e
 
-install -m 755 files/resize2fs_once	"${ROOTFS_DIR}/etc/init.d/"
+if [ "${ENABLE_CLOUD_INIT}" != "1" ]; then
+	# if cloud-init is enabled, it will take care of resizing the rootfs
+	install -m 755 files/resize2fs_once	"${ROOTFS_DIR}/etc/init.d/"
+fi
+
+# TODO: move into conditional block above when cloud-init adds support for commands in final message
+install -m 755 files/rc.local		"${ROOTFS_DIR}/etc/"
 
 install -m 644 files/50raspi		"${ROOTFS_DIR}/etc/apt/apt.conf.d/"
 
 install -m 644 files/console-setup   	"${ROOTFS_DIR}/etc/default/"
 
-install -m 755 files/rc.local		"${ROOTFS_DIR}/etc/"
-
 if [ -n "${PUBKEY_SSH_FIRST_USER}" ]; then
 	install -v -m 0700 -o 1000 -g 1000 -d "${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh
 	echo "${PUBKEY_SSH_FIRST_USER}" >"${ROOTFS_DIR}"/home/"${FIRST_USER_NAME}"/.ssh/authorized_keys
@@ -39,10 +43,6 @@ if [ "${USE_QEMU}" = "1" ]; then
 systemctl disable resize2fs_once
 EOF
 	echo "leaving QEMU mode"
-else
-	on_chroot << EOF
-systemctl enable resize2fs_once
-EOF
 fi
 
 on_chroot <<EOF

stage2/04-cloud-init/00-packages

@@ -0,0 +1,3 @@
+python3-yaml
+netcat-openbsd
+netplan.io

stage2/04-cloud-init/01-run.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -e
+
+if [ "${ENABLE_CLOUD_INIT}" != "1" ]; then
+	log "Skipping cloud-init stage"
+	exit 0
+fi
+
+install -v -D -m 644 -t "${ROOTFS_DIR}/etc/cloud/cloud.cfg.d/" files/99_raspberry-pi.cfg
+
+# install meta-data file for NoCloud data-source to work
+install -v -m 755 files/meta-data "${ROOTFS_DIR}/boot/firmware/meta-data"
+install -v -m 755 files/user-data "${ROOTFS_DIR}/boot/firmware/user-data"
+install -v -m 755 files/network-config "${ROOTFS_DIR}/boot/firmware/network-config"
+
+# setup default netplan config which will instruct netplan to pass control over to network-manager
+# at boot time. This will make NetworkManager manage all devices and by default. 
+# Any Ethernet device will come up with DHCP, once carrier is detected
+install -v -D -m 644 -t "${ROOTFS_DIR}/usr/lib/netplan/" files/00-network-manager-all.yaml
+
+# still does not solve the conflict, maybe some kind of race cond.
+# make sure config stage is run before userconfig service
+#sed -i '/^\[Unit\]/a Before=userconfig.service' "${ROOTFS_DIR}/lib/systemd/system/cloud-config.service"
+
+install -v -m 755 files/cloud-init-custom.deb "${ROOTFS_DIR}/tmp/cloud-init.deb"
+
+# remove cloud-init if already installed for rebuild support while working with custom deb
+on_chroot << EOF
+	SUDO_USER="${FIRST_USER_NAME}" dpkg -i /tmp/cloud-init.deb || true
+	SUDO_USER="${FIRST_USER_NAME}" apt-get install -f -y
+EOF
+
+rm -f "${ROOTFS_DIR}/tmp/cloud-init.deb"
+
+# userconfig service is deleted in export-image/01-user-rename stage

stage2/04-cloud-init/README.txt

@@ -0,0 +1,19 @@
+Cloud-init support for Raspberry Pi OS
+
+TODO: add reference to official documentation for the custom modules when merged
+
+- files/network-config is required because otherwise imager would fail to create the correct filesystem entry
+
+- files/user-data same reason and to include some example configurations
+
+- files/meta-data Cloud-init instance configuration
+
+- files/cloud-init-custom.deb A custom cloud-init build until included apt repositories
+
+- files/99_raspberry-pi.cfg Cloud-init datasource configuration
+
+- files/00-network-manager-all.yaml Example form netplan docs/ubuntu for handing over control from 
+netplan to NetworkManager by default.
+
+Packages:
+    - netplan is installed to provide for advanced options like "wifis" in the network-config v2

stage2/04-cloud-init/files/00-network-manager-all.yaml

@@ -0,0 +1,3 @@
+network:
+  version: 2
+  renderer: NetworkManager

stage2/04-cloud-init/files/99_raspberry-pi.cfg

@@ -0,0 +1,6 @@
+# configure cloud-init with NoCloud
+
+datasource_list: [ NoCloud, None ]
+datasource:
+  NoCloud:
+    fs_label: bootfs

stage2/04-cloud-init/files/meta-data

@@ -0,0 +1,18 @@
+# This is the meta-data configuration file for cloud-init. Please refer to the
+# cloud-init documentation for more information:
+#
+# https://cloudinit.readthedocs.io/
+
+# Set the datasource mode to "local". This ensures that user-data is acted upon
+# prior to bringing up the network (because everything about the datasource is
+# assumed to be local). If you wish to use an HTTP datasource instead, you can
+# change this to "net" or override it on the kernel cmdline (see README).
+dsmode: local
+
+# Specifies the "unique" identifier of the instance. Typically in cloud-init
+# this is generated by the owning cloud and is actually unique (to some
+# degree). Here our data-source is local, so this is just a fixed string.
+# Warning: changing this will cause cloud-init to assume it is running on a
+# "new" instance, and to go through first time setup again (the value is
+# compared to a cached copy).
+instance_id: rpios-image

stage2/04-cloud-init/files/network-config

@@ -0,0 +1,50 @@
+# This file contains a netplan-compatible configuration which cloud-init will
+# apply on first-boot (note: it will *not* update the config after the first
+# boot). Please refer to the cloud-init documentation and the netplan reference
+# for full details:
+#
+# https://netplan.io/reference
+# https://cloudinit.readthedocs.io/en/latest/topics/network-config.html
+# https://cloudinit.readthedocs.io/en/latest/topics/network-config-format-v2.html
+#
+# Please note that the YAML format employed by this file is sensitive to
+# differences in whitespace; if you are editing this file in an editor (like
+# Notepad) which uses literal tabs, take care to only use spaces for
+# indentation. See the following link for more details:
+#
+# https://en.wikipedia.org/wiki/YAML
+#
+# Additionally, please be aware that if your boot sequence depends on active
+# networking (e.g. if your cloud-init configuration pulls packages or SSH
+# keys from the network), you *must* mark at least one interface as required
+# ("optional: false") below. Otherwise, particularly on faster boards,
+# cloud-init will start attempting to use the network before it is ready
+
+# Some additional examples are commented out below
+
+network:
+  version: 2
+
+  ethernets:
+    eth0:
+      dhcp4: true
+      optional: true
+
+#  wifis:
+#    wlan0:
+#      dhcp4: true
+#      optional: true
+#      access-points:
+#        myhomewifi:
+#          password: "S3kr1t"
+#        myworkwifi:
+#          password: "correct battery horse staple"
+#        workssid:
+#          auth:
+#            key-management: eap
+#            method: peap
+#            identity: "me@example.com"
+#            password: "passw0rd"
+#            ca-certificate: /etc/my_ca.pem
+
+#      regulatory-domain: GB

stage2/04-cloud-init/files/user-data

@@ -0,0 +1,109 @@
+#cloud-config
+
+# This is the user-data configuration file for cloud-init. By default this sets
+# up an initial user called "ubuntu" with password "ubuntu", which must be
+# changed at first login. However, many additional actions can be initiated on
+# first boot from this file. The cloud-init documentation has more details:
+#
+# https://cloudinit.readthedocs.io/
+#
+# Please note that the YAML format employed by this file is sensitive to
+# differences in whitespace; if you are editing this file in an editor (like
+# Notepad) which uses literal tabs, take care to only use spaces for
+# indentation. See the following link for more details:
+#
+# https://en.wikipedia.org/wiki/YAML
+#
+# Some additional examples are provided in comments below the default
+# configuration.
+
+# disable_piwiz: fase
+
+# Setup default user
+# rpi_userconf:
+#   password: my-hashed-passwd
+#   user: myusername
+
+## Set the system's hostname. Please note that, unless you have a local DNS
+## setup where the hostname is derived from DHCP requests (as with dnsmasq),
+## setting the hostname here will not make the machine reachable by this name.
+## You may also wish to install avahi-daemon (see the "packages:" key below)
+## to make your machine reachable by the .local domain
+#hostname: raspberrypi
+
+## Set up the keyboard layout. See localectl(1), in particular the various
+## list-x11-* sub-commands, to determine the available models, layouts,
+## variants, and options
+#keyboard:
+#  model: pc105
+#  layout: gb
+#  variant:
+#  options: ctrl:nocaps
+
+# Controls password authentication with the SSH daemon; the default here
+# prevents logging into SSH with a password. Changing this is a security risk
+# and you should at the very least ensure a different default password is
+# specified above
+ssh_pwauth: false
+
+## On first boot, use ssh-import-id to give the specific users SSH access to
+## the default user
+#ssh_import_id:
+#- lp:my_launchpad_username
+#- gh:my_github_username
+
+## Add users and groups to the system, and import keys with the ssh-import-id
+## utility
+#groups:
+#- robot: [robot]
+#- robotics: [robot]
+#- pi
+#
+#users:
+#- default
+#- name: robot
+#  gecos: Mr. Robot
+#  primary_group: robot
+#  groups: users
+#  ssh_import_id: foobar
+#  lock_passwd: false
+#  passwd: $5$hkui88$nvZgIle31cNpryjRfO9uArF7DYiBcWEnjqq7L1AQNN3
+
+## Update apt database and upgrade packages on first boot
+#package_update: true
+#package_upgrade: true
+
+## Install additional packages on first boot
+#packages:
+#- avahi-daemon
+#- rng-tools
+#- python3-gpiozero
+#- [python3-serial, 3.5-1]
+
+## Write arbitrary files to the file-system (including binaries!)
+#write_files:
+#- path: /etc/default/console-setup
+#  content: |
+#    # Consult the console-setup(5) manual page.
+#    ACTIVE_CONSOLES="/dev/tty[1-6]"
+#    CHARMAP="UTF-8"
+#    VIDEOMODE=
+#    FONT="Lat15-Terminus18x10.psf.gz"
+#    FONTFACE=
+#    FONTSIZE=
+#    CODESET="Lat15"
+#  permissions: '0644'
+#  owner: root:root
+#- encoding: gzip
+#  path: /root/Makefile
+#  content: !!binary |
+#    H4sICF2DTWIAA01ha2VmaWxlAFNWCM8syVBILMjPyU/PTC1WKMlXiPB2dlFQNjSx5MpNteLi
+#    dLDiSoRQxYl5KeWZyRkgXrSCkoqKRmaKgm6pppKCbmqhgoFCrIKamkK1QmpyRr6Ckn92YqWS
+#    NdC80uQMBZhOa4VahZoaqIrwjMQSewXfxOxUhcwShcr80qLi1Jw0RSUuAIYfEJmVAAAA
+#  owner: root:root
+#  permissions: '0644'
+
+## Run arbitrary commands at rc.local like time
+#runcmd:
+#- [ ls, -l, / ]
+#- [ sh, -xc, "echo $(date) ': hello world!'" ]