Merge branch 'lf-edge:master' into master

.github/workflows/unit.yml → .github/workflows/go-tests.yml

@@ -1,5 +1,5 @@
 ---
-name: Unit tests
+name: Go Tests
 on:  # yamllint disable-line rule:truthy
   push:
     branches:
@@ -22,7 +22,7 @@ jobs:
       - name: Test
         run: |
           make test
-      - name : Test (TPM Required)
+      - name: Test (TPM Required)
         run: |
           bash tests/tpm/prep-and-test.sh
       - name: Report test results as Annotations

.gitignore

@@ -6,6 +6,7 @@ dist/
 build-tools/bin/
 images/out/
 .go/
+go.work*
 tags
 tmp/
 mkdocs.yml

MAINTAINERS.md

@@ -6,7 +6,7 @@ See [EVE Wiki](https://wiki.lfedge.org/display/EVE/Community) for governance gui
 
 ## List of maintainers (in alphabetical order)
 
-* Avi Deitcher avi@deitcher.net
-* Erik Nordmark erik@zededa.com
-* Milan Lenco milan@zededa.com
-* Roman Shaposhnik rvs@apache.org
+* Avi Deitcher `avi@deitcher.net`
+* Erik Nordmark `erik@zededa.com`
+* Milan Lenco `milan@zededa.com`
+* Roman Shaposhnik `rvs@apache.org`

Makefile

@@ -41,7 +41,7 @@ MEDIA_SIZE=32768
 # Image type for final disk images
 IMG_FORMAT=qcow2
 # Filesystem type for rootfs image
-ROOTFS_FORMAT=squash
+ROOTFS_FORMAT?=squash
 # Image type for installer image
 INSTALLER_IMG_FORMAT=raw
 # Image type for verification image
@@ -392,10 +392,14 @@ currentversion:
 
 test: $(LINUXKIT) test-images-patches | $(DIST)
 	@echo Running tests on $(GOMODULE)
-	$(QUIET)$(DOCKER_GO) "gotestsum --jsonfile $(DOCKER_DIST)/results.json --junitfile $(DOCKER_DIST)/results.xml --raw-command -- go test -coverprofile=coverage.txt -covermode=atomic -race -json ./..." $(GOTREE) $(GOMODULE)
-	$(QUIET)$(DOCKER_GO) "cd \"$(GOTREE)\"; ../../tools/fuzz_test.sh" $(GOTREE) $(GOMODULE)
+	make -C pkg/pillar test
+	cp pkg/pillar/results.json $(DIST)/
+	cp pkg/pillar/results.xml $(DIST)/
 	$(QUIET): $@: Succeeded
 
+test-profiling:
+	make -C pkg/pillar test-profiling
+
 # wrap command into DOCKER_GO and propagate it to the pillar's Makefile
 # for example make pillar-fmt will run docker container based on
 # build-tools/src/scripts/Dockerfile
@@ -641,13 +645,27 @@ $(ROOTFS_TAR): images/out/rootfs-$(HV)-$(PLATFORM).yml | $(INSTALLER)
 	$(QUIET): $@: Begin
 	./tools/makerootfs.sh tar -y $< -t $@ -d $(INSTALLER) -a $(ZARCH)
 	$(QUIET): $@: Succeeded
+ifdef KERNEL_IMAGE
+	# Consider this as a cry from the heart: enormous amount of time is
+	# wasted during kernel rebuild on every small testing change. Now any
+	# kernel image can be used by providing path to a file. You heard it
+	# right: path-to-a-file. No docker. Yay!
+	$(eval KIMAGE = $$(realpath $(KERNEL_IMAGE)))
+	@echo "Replace kernel image in \"$@\" with \"$(KIMAGE)\""
+	# Delete /boot/kernel kernel image
+	tar --delete -f "$@" boot/kernel
+	# Append new kernel image and rename
+	tar -P -u --transform="flags=r;s|$(KIMAGE)|/boot/kernel|" -f "$@" "$(KIMAGE)"
+endif
 
 $(ROOTFS_IMG): $(ROOTFS_TAR) | $(INSTALLER)
 	$(QUIET): $@: Begin
 	./tools/makerootfs.sh imagefromtar -t $(ROOTFS_TAR) -i $@ -f $(ROOTFS_FORMAT) -a $(ZARCH)
 	@echo "size of $@ is $$(wc -c < "$@")B"
+ifeq ($(ROOTFS_FORMAT),squash)
 	@[ $$(wc -c < "$@") -gt $$(( $(ROOTFS_MAXSIZE_MB) * 1024 * 1024 )) ] && \
 	        echo "ERROR: size of $@ is greater than $(ROOTFS_MAXSIZE_MB)MB (bigger than allocated partition)" && exit 1 || :
+endif
 	$(QUIET): $@: Succeeded
 
 sbom_info:
@@ -665,11 +683,6 @@ $(SBOM): $(ROOTFS_TAR) | $(INSTALLER)
 	# this all can go away, and we can read the rootfs.tar
 	# see https://github.com/anchore/syft/issues/1400
 	tar xf $< -C $(TMP_ROOTDIR) --exclude "dev/*"
-	# kernel-*.spdx.json are now generated in eve-kernel repo and are stored in docker  image.
-	# Manually extract them to unpacked rootfs.
-	# Later linuxkit will get a support for SBOM in OCI metadata and this step as well as manual run of
-	# syft will be deprecated
-	docker export $(shell docker create $(KERNEL_TAG) create) | tar xv -C $(TMP_ROOTDIR) --wildcards --no-anchored '*.spdx.json'
 	docker run -v $(TMP_ROOTDIR):/rootdir:ro -v $(CURDIR)/.syft.yaml:/syft.yaml:ro $(SYFT_IMAGE) -c /syft.yaml --base-path /rootdir /rootdir > $@
 	rm -rf $(TMP_ROOTDIR)
 	$(QUIET): $@: Succeeded
@@ -713,12 +726,12 @@ publish_sources: $(COLLECTED_SOURCES)
 
 $(LIVE).raw: $(BOOT_PART) $(EFI_PART) $(ROOTFS_IMG) $(CONFIG_IMG) $(PERSIST_IMG) $(BSP_IMX_PART) | $(INSTALLER)
 	./tools/prepare-platform.sh "$(PLATFORM)" "$(BUILD_DIR)" "$(INSTALLER)" || :
-	./tools/makeflash.sh "mkimage-raw-efi" -C 559 $| $@ $(PART_SPEC)
+	./tools/makeflash.sh "mkimage-raw-efi" -C $| $@ $(PART_SPEC)
 	$(QUIET): $@: Succeeded
 
 $(INSTALLER).raw: $(BOOT_PART) $(EFI_PART) $(ROOTFS_IMG) $(INITRD_IMG) $(INSTALLER_IMG) $(CONFIG_IMG) $(PERSIST_IMG) $(BSP_IMX_PART) | $(INSTALLER)
 	./tools/prepare-platform.sh "$(PLATFORM)" "$(BUILD_DIR)" "$(INSTALLER)" || :
-	./tools/makeflash.sh "mkimage-raw-efi" -C 592 $| $@ "conf_win installer inventory_win"
+	./tools/makeflash.sh "mkimage-raw-efi" -C $| $@ "conf_win installer inventory_win"
 	$(QUIET): $@: Succeeded
 
 $(INSTALLER).iso: $(EFI_PART) $(ROOTFS_IMG) $(INITRD_IMG) $(INSTALLER_IMG) $(CONFIG_IMG) $(PERSIST_IMG) | $(INSTALLER)
@@ -741,7 +754,7 @@ $(LIVE).parallels: $(LIVE).raw
 
 $(VERIFICATION).raw: $(BOOT_PART) $(EFI_PART) $(ROOTFS_IMG) $(INITRD_IMG) $(VERIFICATION_IMG) $(CONFIG_IMG) $(PERSIST_IMG) $(BSP_IMX_PART) | $(VERIFICATION)
 	./tools/prepare-platform.sh "$(PLATFORM)" "$(BUILD_DIR)" "$(VERIFICATION)" || :
-	./tools/makeflash.sh "mkverification-raw-efi" -C 850 $| $@ "conf_win verification inventory_win"
+	./tools/makeflash.sh "mkverification-raw-efi" -C $| $@ "conf_win verification inventory_win"
 	$(QUIET): $@: Succeeded
 
 $(VERIFICATION).net: $(EFI_PART) $(ROOTFS_TAR) $(ROOTFS_IMG) $(INITRD_IMG) $(VERIFICATION_IMG) $(CONFIG_IMG) $(PERSIST_IMG) | $(VERIFICATION)
@@ -834,6 +847,9 @@ cache-export-docker-load-all: $(LINUXKIT) $(addsuffix -cache-export-docker-load,
 proto-vendor:
 	@$(DOCKER_GO) "cd pkg/pillar ; go mod vendor" $(CURDIR) proto
 
+bump-eve-api:
+	find . -type f -name "go.mod" -exec grep -q 'github.com/lf-edge/eve-api/go' {} \; -execdir go get -u github.com/lf-edge/eve-api/go \; -execdir go mod tidy \; -execdir go mod vendor \;
+
 .PHONY: proto-api-%
 
 check-patch-%:
@@ -1009,6 +1025,7 @@ help:
 	@echo "Commonly used maintenance and development targets:"
 	@echo "   build-vm       prepare a build VM for EVE in qcow2 format"
 	@echo "   test           run EVE tests"
+	@echo "   test-profiling run pillar tests with memory profiler"
 	@echo "   clean          clean build artifacts in a current directory (doesn't clean Docker)"
 	@echo "   release        prepare branch for a release (VERSION=x.y.z required)"
 	@echo "   patch          make a patch release on a current branch (must be a release branch)"
@@ -1017,6 +1034,9 @@ help:
 	@echo "   shell          drop into docker container setup for Go development"
 	@echo "   yetus          run Apache Yetus to check the quality of the source tree"
 	@echo
+	@echo "Seldom used maintenance and development targets:"
+	@echo "   bump-eve-api   bump eve-api in all subprojects"
+	@echo
 	@echo "Commonly used build targets:"
 	@echo "   build-tools      builds linuxkit utilities and installs under build-tools/bin"
 	@echo "   config           builds a bundle with initial EVE configs"

build-tools/src/scripts/Dockerfile

@@ -9,17 +9,53 @@ ARG GID
 # this must be an ARG so it doesn't carry through post-build phase
 ARG all_proxy
 # hadolint ignore=DL3018
-RUN apk add --no-cache openssh-client git gcc linux-headers libc-dev util-linux libpcap-dev bash vim make protobuf protobuf-dev sudo tar curl graphviz ttf-freefont patch dnsmasq
-# we need updated libraries, here we use the same version as for eve/alpine
-# hadolint ignore=DL3018
-RUN apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/v3.16/main add -U --upgrade zfs-dev zfs-libs
+RUN apk add --no-cache openssh-client git gcc linux-headers libc-dev util-linux libpcap-dev bash vim make protobuf \
+        protobuf-dev sudo tar curl graphviz ttf-freefont patch dnsmasq ca-certificates build-base gettext-dev \
+        libtirpc-dev automake autoconf libtool attr-dev e2fsprogs-dev glib-dev openssl-dev util-linux-dev
+
 RUN deluser ${USER} ; delgroup ${GROUP} || :
 RUN sed -ie /:${UID}:/d /etc/passwd /etc/shadow ; sed -ie /:${GID}:/d /etc/group || :
 RUN addgroup -g ${GID} ${GROUP} && adduser -h /home/${USER} -G ${GROUP} -D -H -u ${UID} ${USER}
 RUN echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER}
-# dep is deprecated and probably should be gotten rid of; no need to parametrize the version, as there will be no further releases
-# hadolint ignore=SC2086,DL4006
-RUN OS="$(uname -o | tr '[:upper:]' '[:lower:]')" && PLATFORM="$(go version | sed 's#^.*'${OS}'/##g')" && curl -o /usr/local/bin/dep -L "https://github.com/golang/dep/releases/download/v0.5.4/dep-${OS}-${PLATFORM}" && chmod +x /usr/local/bin/dep
+### Build OpenZFS 2.2.2 libs here for 'make test' and full eve build in one spot
+# should be aligned with kernel
+#  * ZFS on Linux
+# coreutils's uname -o breaks above url generation.
+# hadolint ignore=DL3018
+RUN apk add --no-cache coreutils
+ENV ZFS_VERSION=2.2.2
+ENV ZFS_COMMIT=zfs-${ZFS_VERSION}
+ENV ZFS_REPO=https://github.com/openzfs/zfs
+
+WORKDIR /tmp/zfs
+# hadolint ignore=DL3020
+ADD ${ZFS_REPO}/tarball/${ZFS_COMMIT}/ zfs.tgz
+RUN tar -zxvf zfs.tgz  --strip-components=1 && \
+    rm zfs.tgz
+
+RUN ./autogen.sh && \
+    ./configure \
+    --prefix=/usr \
+    --with-tirpc \
+    --sysconfdir=/etc \
+    --mandir=/usr/share/man \
+    --infodir=/usr/share/info \
+    --localstatedir=/var \
+    --with-config=user \
+    --with-udevdir=/lib/udev \
+    --disable-systemd \
+    --disable-static && \
+    ./scripts/make_gitrev.sh && \
+    make -j "$(getconf _NPROCESSORS_ONLN)" && \
+    make DESTDIR=/tmp/zfs-out install-strip
+
+# cleanup
+RUN rm -rf /tmp/zfs-out/usr/share && rm -rf /tmp/zfs-out/usr/src && \
+    rm -rf /tmp/zfs-out/etc/init.d && rm -rf /tmp/zfs-out/etc/conf.d
+
+RUN cp -r /tmp/zfs-out/* /
+
+
 RUN go install github.com/golang/protobuf/protoc-gen-go@v1.5.2
 RUN go install gotest.tools/gotestsum@v1.7.0
 RUN go install github.com/seamia/protodot@87817c3d0a8e7af753af15508b51292e941bc7c6

docs/BOOTING.md

@@ -132,7 +132,7 @@ in grub.cfg with graphical GRUB menu to get the device to boot again.
     11. `eve_install_skip_rootfs` - do not install rootfs partition onto device. May be selected from graphical GRUB menu.
     12. `eve_install_skip_zfs_checks` - install zfs by skipping minimum requirement checks.
     13. `eve_install_zfs_with_raid_level` - Sets raid level for zfs storage. Valid values are none,raid1,raid5,raid6. Default value is none. This option also applied for the first boot of a live image to prepare zfs persist pool instead of ext4.
-    14. `eve_install_kubevirt_reserve_for_eve_sizeGB` - Amount of space in GB to reserve for eve services in kubevirt based images (This is highly experimental and not supported config, also its an optional parameter and defaults to 20GB if not set)
+    14. `eve_install_kubevirt_etcd_sizeGB` - Size in GB of the etcd-storage zvol.  Defaults to 10GB.
 3. General kernel parameters may be adjusted with `set_global dom0_extra_args "$dom0_extra_args OPTION1=VAL1 OPTION2 "`.
    They will be added to kernel cmdline.
 

docs/BUILD.md

@@ -237,11 +237,12 @@ To build `live.raw`:
 
 1. Ensure `rootfs.img` and `config.img` are ready.
 2. tar these two dependencies together and stream to [makeflash.sh](../makeflash.sh)
-3. Call `./makeflash.sh -C <disksize> live.raw`, where _disksize_ is normally 8192MB
+3. Call `./makeflash.sh -C live.raw`
 4. `makeflash.sh` creates an empty image of the target size at the target path, and then passes control to a docker container from the image generated by `pkg/mkimage-raw-efi`. It does _not_ pass on the names of the partitions to be created, depending on the default.
 5. `mkimage-raw-efi`:
-    1. extracts the contents of the tar stream to `/parts/`
-    2. creates a partition for each of `efi`, `imga`, `imgb`, `conf`, `persist`
+    0. Calculates required size of the resulting image if '-C' option is provided
+    1. Extracts the contents of the tar stream to `/parts/`
+    2. Creates a partition for each of `efi`, `imga`, `imgb`, `conf`, `persist`
     3. Populates each partition with its appropriate contents:
         * `efi`: contents of `/EFI/BOOT/` from `/parts/rootfs.img`
         * `imga`/`imgb`: contents of `/parts/rootfs.img`
@@ -281,11 +282,11 @@ For an installable image, named `installer.img`, we create the following depende
 
 #### Installer: rootfs.img
 
-`rootfs.img` is built identically to how it is for a live bootable image, see [rootfs.img](#rootfs.img)
+`rootfs.img` is built identically to how it is for a live bootable image, see [rootfs.img](#rootfsimg)
 
 #### Installer: config.img
 
-`config.img` is built identically to how it is for a live bootable image, see [config.img](#config.img)
+`config.img` is built identically to how it is for a live bootable image, see [config.img](#configimg)
 
 #### Installer: rootfs_installer.img
 
@@ -308,11 +309,12 @@ To build `installer.raw`:
 
 1. Ensure `rootfs_installer.img` and `config.img` are ready.
 2. tar these two dependencies together and stream to [makeflash.sh](../makeflash.sh)
-3. Call `./makeflash.sh -C <disksize> live.raw "efi imga conf_win"`, where _disksize_ is normally 350MB
+3. Call `./makeflash.sh -C live.raw "efi imga conf_win"`
 4. `makeflash.sh` creates an empty image of the target size at the target path, and then passes control to a docker container from the image generated by `pkg/mkimage-raw-efi`. It _does_ pass on the names of the partitions to be created, limiting it to `efi`, `imga`, `conf_win`. There is no need for `imgb` or `persist` partitions for an installer image that will not be persisting data, and will not be updating its root filesystem.
 5. `mkimage-raw-efi`:
-    1. extracts the contents of the tar stream to `/parts/`
-    2. creates a partition for each of `efi`, `imga`, `conf_win`
+    0. Calculates required size of the resulting image if '-C' option is provided
+    1. Extracts the contents of the tar stream to `/parts/`
+    2. Creates a partition for each of `efi`, `imga`, `conf_win`
     3. Populates each partition with its appropriate contents:
         * `efi`: contents of `/EFI/BOOT/` from `/parts/rootfs.img`
         * `imga`: contents of `/parts/rootfs.img`