balena-io-experimental · ab77 · Feb 28, 2022
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
+.DS_Store
 node_modules
diff --git a/balena.sh b/balena.sh
@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2154,SC2034,SC1090
+set -ea
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+if [[ -n "${BALENA_DEVICE_UUID}" ]]; then
+    # prepend the device UUID if running on balenaOS
+    TLD="${BALENA_DEVICE_UUID}.${DNS_TLD}"
+else
+    TLD="${DNS_TLD}"
+fi
+
+BALENA_API_URL=${BALENA_API_URL:-https://api.balena-cloud.com}
+certs=${CERTS:-/certs}
+conf=${CONF:-/balena/${TLD}.env}
+test_fleet=${TEST_FLEET:-test-fleet}
+device_type=${DEVICE_TYPE:-qemux86-64}
+os_version=${OS_VERSION:-$(balena os versions "${device_type}" | grep \.dev | head -n 1)}
+guest_disk_size=${GUEST_DISK_SIZE:-16}
+guest_image=${GUEST_IMAGE:-/balena/balena.img}
+attempts=${ATTEMPTS:-3}
+
+function set_update_lock {
+    if [[ -n $BALENA_SUPERVISOR_ADDRESS ]] && [[ -n $BALENA_SUPERVISOR_API_KEY ]]; then
+        while [[ $(curl --silent --retry "${attempts}" --fail \
+          "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+          -H "Content-Type: application/json" | jq -r '.update_pending') == 'true' ]]; do
+
+            curl --silent --retry "${attempts}" --fail \
+              "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+              -H "Content-Type: application/json" | jq -r
+
+            sleep "$(( (RANDOM % 3) + 3 ))s"
+        done
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+
+        # https://www.balena.io/docs/learn/deploy/release-strategy/update-locking/
+        lockfile /tmp/balena/updates.lock
+    fi
+}
+
+function remove_update_lock() {
+    rm -f /tmp/balena/updates.lock
+}
+
+function cleanup() {
+   rm -f /tmp/balena.zip
+   remove_update_lock
+
+   # crash loop backoff
+   sleep 10s
+}
+
+trap 'cleanup' EXIT
+
+function update_ca_certificates() {
+    # only set CA bundle if using private certificate chain
+    if [[ -e "${certs}/ca-bundle.pem" ]]; then
+        if [[ "$(readlink -f "${certs}/${TLD}-chain.pem")" =~ \/private\/ ]]; then
+            mkdir -p /usr/local/share/ca-certificates
+            cat < "${certs}/ca-bundle.pem" > /usr/local/share/ca-certificates/balenaRootCA.crt
+            # shellcheck disable=SC2034
+            CURL_CA_BUNDLE=${CURL_CA_BUNDLE:-${certs}/ca-bundle.pem}
+            NODE_EXTRA_CA_CERTS=${NODE_EXTRA_CA_CERTS:-${CURL_CA_BUNDLE}}
+            # (TBC) refactor to use NODE_EXTRA_CA_CERTS instead of ROOT_CA
+            # https://github.com/balena-io/e2e/blob/master/conf.js#L12-L14
+            # https://github.com/balena-io/e2e/blob/master/Dockerfile#L82-L83
+            # ... or
+            # https://thomas-leister.de/en/how-to-import-ca-root-certificate/
+            # https://github.com/puppeteer/puppeteer/issues/2377
+            ROOT_CA=${ROOT_CA:-$(cat < "${NODE_EXTRA_CA_CERTS}" | openssl base64 -A)}
+        else
+            rm -f /usr/local/share/ca-certificates/balenaRootCA.crt
+            unset NODE_EXTRA_CA_CERTS CURL_CA_BUNDLE ROOT_CA
+        fi
+        update-ca-certificates
+    fi
+}
+
+function wait_for_api() {
+    while ! curl --silent --fail "https://api.${DNS_TLD}/ping"; do
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+}
+
+function open_balena_login() {
+    balena login --credentials \
+      --email "${SUPERUSER_EMAIL}" \
+      --password "${SUPERUSER_PASSWORD}"
+}
+
+function create_fleet() {
+    if ! balena fleet "${test_fleet}"; then
+        # wait for API to load DT contracts
+        # (TBC) 'balena devices supported' always returns empty list
+        while ! balena fleet create "${test_fleet}" --type "${device_type}"; do
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+    fi
+}
+
+function download_os_image() {
+    if ! [[ -e $guest_image ]]; then
+        wget -qO /tmp/balena.zip \
+          "${BALENA_API_URL}/download?deviceType=${device_type}&version=${os_version:1}&fileType=.zip&developmentMode=true"
+
+        unzip -oq /tmp/balena.zip -d /tmp
+
+        cat < "$(find /tmp -type f -name "*.img" | head -n 1)" > "${guest_image}"
+    fi
+}
+
+function configure_virtual_device() {
+    while ! [[ -e $guest_image ]]; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+
+    if ! [[ -e /balena/config.json ]]; then
+        balena_device_uuid="$(openssl rand -hex 16)"
+
+        balena device register "${test_fleet}" --uuid "${balena_device_uuid}"
+
+        balena config generate \
+          --version "${os_version:1}" \
+          --device "${balena_device_uuid}" \
+          --network ethernet \
+          --appUpdatePollInterval 10 \
+          --output /balena/config.json
+    fi
+
+    balena os configure "${guest_image}" \
+      --fleet "${test_fleet}" \
+      --config /balena/config.json
+
+}
+
+function resize_disk_image() {
+    if ! [[ -e /balena/standard0.qcow2 ]]; then
+        qemu-img convert -f raw -O qcow2 \
+          "${guest_image}" \
+          "/balena/standard0.qcow2"
+
+        qemu-img resize "/balena/standard0.qcow2" "${guest_disk_size}G"
+    fi
+}
+
+function convert_raw_image() {
+    if ! [[ -e /balena/standard0-snapshot.qcow2 ]]; then
+        qemu-img create \
+          -f qcow2 -b "/balena/standard0.qcow2" \
+          -F qcow2 "/balena/standard0-snapshot.qcow2" \
+          "$(( guest_disk_size / 2 ))G"
+    fi
+}
+
+function enable_nested_virtualisation() {
+    if modprobe kvm_intel; then
+        echo 1 > /sys/kernel/mm/ksm/run
+    else
+        sed -i '/accel: kvm/d' guests.yml
+    fi
+}
+
+if [[ $TLD =~ ^.*\.local\.? ]]; then
+    echo 'mDNS configurations not supported'
+    sleep infinity
+fi
+
+[[ -f $conf ]] && source "${conf}"
+
+BALENARC_BALENA_URL="${DNS_TLD}"
+
+enable_nested_virtualisation
+update_ca_certificates
+wait_for_api
+balena whoami || open_balena_login
+create_fleet
+download_os_image
+configure_virtual_device
+
+set_update_lock
+resize_disk_image
+convert_raw_image
+remove_update_lock
+
+exec /root/cli.js "$@"
diff --git a/cli/Dockerfile b/cli/Dockerfile
@@ -9,23 +9,55 @@ RUN apk add --update --no-cache \
 
 WORKDIR /root
 
+RUN apk add --no-cache \
+	bash \
+	curl \
+	docker-cli \
+	jq \
+	minicom \
+	netcat-openbsd \
+	openssh-client \
+	qemu-img \
+	qemu-system-x86_64 \
+	unzip \
+	wget \
+	&& apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.11/main \
+	procmail
+
+
+# --- build
 FROM base AS build
 
-# install npm in a separate build stage to save space in the runtime image
-RUN apk add --update --no-cache npm
+RUN apk add --no-cache -t .build-deps \
+    build-base \
+    git \
+    linux-headers \
+    python3
+
+COPY run-tests.sh package*.json *.js ./
+
+RUN npm i && apk del --purge .build-deps
 
-COPY package.json *.js ./
-RUN npm i
 
+# --- runtime
 FROM base AS run
 
+WORKDIR /data/
+
 COPY --from=build /root/ /root/
 
+COPY *.yml ./
+
+COPY docker-hc balena.sh /usr/sbin/
+
+RUN ln -sf /root/node_modules/balena-cli/bin/balena /usr/bin/balena
+
 # create qemu-bridge-helper ACL file
 # https://wiki.qemu.org/Features/HelperNetworking
 RUN mkdir -p /etc/qemu \
 	&& echo "allow all" > /etc/qemu/bridge.conf \
-	&& chmod 0640 /etc/qemu/bridge.conf
+	&& chmod 0640 /etc/qemu/bridge.conf \
+	&& addgroup root qemu \
+	&& addgroup root kvm
 
-WORKDIR /data/
 CMD /root/cli.js
diff --git a/cli/package.json b/cli/package.json
@@ -11,6 +11,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "balena-cli": "^13.2.1",
     "mz": "^2.7.0",
     "yaml": "^1.10.2",
     "yargs": "^17.5.0"

diff --git a/docker-hc b/docker-hc
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2154,SC2034,SC1090
+set -ea
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+conf=${CONF:-/balena/${BALENA_DEVICE_UUID}.${DNS_TLD}.env}
+test_fleet=${TEST_FLEET:-test-fleet}
+
+function open_balena_login() {
+	balena login --credentials \
+	  --email "${SUPERUSER_EMAIL}" \
+	  --password "${SUPERUSER_PASSWORD}"
+}
+
+function check_device_status() {
+	if [[ -e /balena/config.json ]]; then
+		balena_device_uuid="$(cat < /balena/config.json | jq -r .uuid)"
+
+		if [[ -n $balena_device_uuid ]]; then
+			is_online="$(balena devices --json --fleet "${test_fleet}" \
+			  | jq -r --arg uuid "${balena_device_uuid}" '.[] | select(.uuid==$uuid).is_online == true')"
+
+			if [[ $is_online =~ true ]]; then
+				exit 0
+			else
+				exit 1
+			fi
+		fi
+	fi
+}
+
+[[ -f $conf ]] && source "${conf}"
+
+BALENARC_BALENA_URL="${DNS_TLD}"
+
+balena whoami || open_balena_login
+
+if [[ $TLD =~ ^.*\.local\.? ]]; then
+    # mDNS configurations not supported
+    exit 0
+else
+    check_device_status
+fi
diff --git a/guests.yml b/guests.yml
@@ -0,0 +1,49 @@
+# ~ https://github.com/balena-io/autohat/blob/master/resources/qemu.robot
+templates:
+  standard:
+    machine:
+      - type: q35
+        # (TBC) not supported on AWS/EC2 unless using metal instance classes|types
+        # only supported on AMIs build on AWS Nitro System
+        accel: kvm
+    smp: cores=2
+    m: 512M
+    drive:
+      - file: /balena/standard0.qcow2
+        format: qcow2
+        if: none
+        index: 0
+        media: disk
+        id: disk
+    device:
+      - ahci:
+        id: ahci
+      - ide-hd:
+        drive: disk
+        bus: ahci.0
+      - virtio-net-pci:
+        netdev: n1
+    netdev:
+      #- "user,id=n1,dns=127.0.0.1,guestfwd=tcp:10.0.2.100:80-cmd:netcat haproxy 80,tcp:10.0.2.100:443-cmd:netcat haproxy 443"
+      - user:
+        id: n1
+        dns: 127.0.0.1
+        guestfwd:
+          # (TBC) escape spaces in command
+          #- tcp:10.0.2.100:80-cmd:nc haproxy 80
+          - tcp:10.0.2.100:443-cmd:nc haproxy 443
+    # (minicom) https://github.com/balena-io/autohat#troubleshooting
+    chardev:
+      - socket:
+        id: serial0
+        path: /tmp/console.sock
+        server: on
+        wait: off
+    serial: chardev:serial0
+    monitor: none
+    nographic:
+
+guests:
+  - template: standard
+    arch: x86_64
+    count: 1