This repository has been archived by the owner on Oct 2, 2024. It is now read-only.

several fixes related to `-W` testing #3000

Workflow file for this run

	name: test suite

	on:
	pull_request: # all pull requests
	push:
	branches: [ master ] # all commits on master
	schedule:
	- cron: '0 2 * * 0' # every Sunday at 2:00am UTC (Saturday 7:00pm MST)

	jobs:
	main:
	runs-on: ubuntu-22.04
	timeout-minutes: 90
	strategy:
	fail-fast: false # Actions seems kind of flaky lately
	matrix:
	builder:
	- ch-image
	pack_fmt: [squash-mount, tar-unpack, squash-unpack]
	keep_sudo: # if false, remove self from sudoers post-install/setup
	- false
	cache:
	- enabled
	include:
	- builder: none
	pack_fmt: squash-mount
	keep_sudo: false
	cache: disabled
	- builder: docker
	pack_fmt: tar-unpack
	keep_sudo: true
	cache: enabled
	- builder: ch-image
	pack_fmt: squash-mount
	keep_sudo: false
	cache: disabled

	env:
	CH_IMAGE_CACHE: ${{ matrix.cache }}
	CH_TEST_BUILDER: ${{ matrix.builder }}
	CH_TEST_TARDIR: /mnt/tarballs
	CH_TEST_IMGDIR: /mnt/images
	CH_TEST_PACK_FMT: ${{ matrix.pack_fmt }}
	CH_TEST_PERMDIRS: /mnt/perms_test /run/perms_test
	ch_prefix: /var/tmp

	steps:
	- uses: actions/checkout@v3

	- name: early setup & validation
	run: \|
	[[ -n $CH_TEST_BUILDER ]]
	sudo timedatectl set-timezone America/Denver
	sudo chmod 1777 /mnt /usr/local/src
	echo "ch_makej=-j$(getconf _NPROCESSORS_ONLN)" >> $GITHUB_ENV
	# Remove sbin directories from $PATH (see issue #43). Assume none of
	# these are the first entry in $PATH.
	echo "PATH=$PATH"
	path_new=$PATH
	for i in /sbin /usr/sbin /usr/local/sbin; do
	path_new=${path_new/:$i/}
	done
	echo "path_new=$path_new"
	echo "PATH=$path_new" >> $GITHUB_ENV
	# Set sudo umask to something quite restrictive. The default is
	# 0022, but we had a "make install" bug (issue #947) that was
	# tickled by 0027, which is a better setting. For reasons I don’t
	# understand, this only affects sudo, but since that’s what we want,
	# I left it. Note there are a few dependency installs below that
	# have similar permissions bugs; these relax the umask on a
	# case-by-case basis.
	sudo sed -i -E 's/^UMASK\s+022/UMASK 0077/' /etc/login.defs
	fgrep UMASK /etc/login.defs

	- name: print starting environment
	run: \|
	echo builder: ${{ matrix.builder }}
	echo pack_fmt: ${{ matrix.pack_fmt }}
	echo keep_sudo: ${{ matrix.keep_sudo }}
	echo cache: ${{ matrix.cache }}
	uname -a
	lsb_release -a
	id
	pwd
	getconf _NPROCESSORS_ONLN
	free -m
	df -h
	locale -a
	timedatectl
	env \| egrep '^(PATH\|USER)='
	env \| egrep '^(ch\|CH)_'
	[[ $PATH != /usr/local/sbin ]] # verify sbin removal; see above
	printf 'umask for %s: ' $USER && umask
	printf 'umask under sudo: ' && sudo sh -c umask
	[[ $(umask) = 0022 ]]
	[[ $(sudo sh -c umask) = 0077 ]]

	- name: lines of code
	if: ${{ matrix.builder == 'none' }}
	run: \|
	sudo apt-get install cloc
	misc/loc

	- name: install/configure dependencies, all
	run: \|
	# configure doesn’t tell us about these.
	sudo apt-get install attr pigz pv
	# configure does tell us about these.
	sudo apt-get install bats squashfs-tools
	# Track newest Sphinx in case it breaks things.
	# FIXME: pip yells about docutils 0.19 so ask for latest 0.18
	sudo su -c 'umask 0022 && pip3 install docutils==0.18.1 sphinx sphinx-rtd-theme sphinx-reredirects'
	# Use newest shellcheck.
	cd /usr/local/src
	wget -nv https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz
	tar xf shellcheck-stable.linux.x86_64.tar.xz
	sudo mv shellcheck-stable/shellcheck /usr/local/bin
	which shellcheck
	shellcheck --version

	- name: install/configure dependencies, ch-image
	if: ${{ matrix.builder == 'ch-image' }}
	run: \|
	# Use most current packages b/c new versions sometimes break things.
	sudo sh -c 'umask 0022 && pip3 install lark requests wheel'
	# Configure Git and friends for build cache.
	if [[ $CH_IMAGE_CACHE = enabled ]]; then
	# Git
	command -v git
	git --version
	git config --global user.name 'Judit Polgár'
	git config --global user.email judit@example.com
	git config --global core.excludesfile ~/.gitignore
	echo __ch-test_ignore__ >> ~/.gitignore
	# Graphviz
	sudo apt-get install -y graphviz
	dot -V
	# git2dot
	sudo sh -c 'umask 0022 && pip install python-dateutil'
	git clone https://github.com/jlinoff/git2dot.git ~/git2dot
	cd ~/git2dot
	git checkout $(git tag \| sort -Vr \| head -1)
	sudo cp -a git2dot.py /usr/local/bin
	command -v git2dot.py
	git2dot.py --version
	fi

	- name: install libsquashfuse
	if: ${{ matrix.pack_fmt == 'squash-mount' }}
	run: \|
	set -x
	sudo apt-get install -y libfuse3-dev
	cd /usr/local/src
	git clone https://github.com/vasi/squashfuse.git
	cd squashfuse
	# Check out the latest release.
	git checkout $(git tag -l '[0-9].*' \| sort -V \| tail -1)
	./autogen.sh
	./configure --prefix=/usr/local
	make
	sudo sh -c 'umask 0022 && make install'
	sudo ldconfig
	# Unset setuid on all fusermount3(1) on the system.
	sudo chmod -v u-s /usr/bin/fusermount*
	ls -lh $(command -v fusermount3)
	[[ ! -u $(command -v fusermount3) ]]


	- name: install musl and friends
	if: ${{ matrix.pack_fmt == 'tar-unpack' }}
	run: \|
	sudo apt-get install -y musl-tools
	# At present we use argp to parse command line arguments. This is
	# not portable; it’s a glibc extension (see #1260). Thus, install a
	# standalone argp.
	cd /usr/local/src
	git clone --depth=1 https://github.com/ericonr/argp-standalone.git
	cd argp-standalone
	autoreconf --force --install
	CC=musl-gcc ./configure
	make $ch_makej
	sudo install -D -m644 argp.h /usr/include/x86_64-linux-musl/argp.h
	sudo install -D -m755 libargp.a /usr/lib/x86_64-linux-musl/libargp.a
	# musl-gcc does not install Linux headers. This is a bad kludge.
	# See: https://www.openwall.com/lists/musl/2017/11/23/1
	sudo ln -s ../linux ../asm-generic ../x86_64-linux-gnu \
	/usr/include/x86_64-linux-musl/
	sudo ln -s x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/

	- name: disable bundled lark, ch-image
	if: ${{ matrix.builder == 'ch-image'
	&& matrix.pack_fmt == 'squash-mount'
	&& matrix.cache == 'disabled' }}
	run: \|
	set -x
	# no install, disable
	./autogen.sh --no-lark
	./configure --disable-bundled-lark --prefix=$ch_prefix/from-git
	fgrep '"lark" module ... external' config.log
	! test -f ./lib/lark/lark.py
	make $ch_makej
	sudo make $ch_makej install
	[[ $(bin/ch-image -v --dependencies) = 'lark path: /usr/local/lib/python3.10/dist-packages/lark/__init__.py' ]]
	[[ $($ch_prefix/from-git/bin/ch-image -v --dependencies) = 'lark path: /usr/local/lib/python3.10/dist-packages/lark/__init__.py' ]]
	make clean
	# install, disable
	./autogen.sh
	./configure --disable-bundled-lark --prefix=$ch_prefix/from-git
	fgrep '"lark" module ... bundled' config.log
	test -f ./lib/lark/lark.py
	make $ch_makej
	sudo make $ch_makej install
	[[ $(bin/ch-image -v --dependencies) = 'lark path: /home/runner/work/charliecloud/charliecloud/lib/lark/__init__.py' ]]
	[[ $($ch_prefix/from-git/bin/ch-image -v --dependencies) = 'lark path: /usr/local/lib/python3.10/dist-packages/lark/__init__.py' ]]
	make clean
	# no install, enable: build fails, untested
	# install, enable: normal configuration, remainder of CI

	- name: build/install from Git
	run: \|
	./autogen.sh
	# Remove Autotools to make sure everything works without them.
	sudo apt-get remove autoconf automake
	# Configure and verify output.
	#
	# We want to support any standard libc, so build against musl for
	# some cases:
	#
	# squash-mount: glibc; libsquashfuse and libfuse3 are built with it
	# squash-unpack: glibc; to test ch-fromhost which assumes glibc
	# tar-unpack: musl
	#
	# See PR #1258, which is a FTBFS for musl. Other build steps in this
	# workflow also use glibc.
	if [[ $CH_TEST_PACK_FMT == tar-unpack ]]; then
	export CC=musl-gcc
	# GNU ldd(1) doesn’t work on musl binaries.
	LDD='/lib/ld-musl-x86_64.so.1 --list'
	else
	LDD=ldd
	fi
	# Go wild with install directories to make sure it works (see #683).
	# We do have a normal install below.
	./configure --prefix=$ch_prefix/from-git \
	--datarootdir=$ch_prefix/from-git_dataroot \
	--libexecdir=$ch_prefix/from-git_libexec \
	--libdir=$ch_prefix/from-git_lib \
	--mandir=$ch_prefix/from-git_man \
	--docdir=$ch_prefix/from-git_doc \
	--htmldir=$ch_prefix/from-git_html
	set -x
	fgrep 'documentation: yes' config.log
	[[ $CH_TEST_BUILDER = docker ]] && command -v docker
	if [[ $CH_TEST_BUILDER = ch-image ]]; then
	fgrep 'with ch-image(1): yes' config.log
	fgrep '"lark" module ... bundled' config.log
	test -f ./lib/lark/lark.py
	fi
	fgrep 'recommended tests, tar-unpack mode: yes' config.log
	fgrep 'recommended tests, squash-unpack mode: yes' config.log
	if [[ $CH_TEST_PACK_FMT = squash-mount ]]; then
	fgrep 'recommended tests, squash-mount mode: yes' config.log
	fgrep 'internal SquashFS mounting ... yes' config.log
	else
	fgrep 'recommended tests, squash-mount mode: no' config.log
	fgrep 'internal SquashFS mounting ... no' config.log
	fi
	set +x
	# Build and install.
	make $ch_makej
	sudo make $ch_makej install
	$LDD bin/ch-run
	bin/ch-run --version
	$LDD $ch_prefix/from-git/bin/ch-run
	$ch_prefix/from-git/bin/ch-run --version
	# Ensure bundled Lark in tarball.
	make $ch_makej dist
	tar tf charliecloud-*.tar.gz \| fgrep /lib/lark/lark.py

	- name: late setup & validation, ch-image
	if: ${{ matrix.builder == 'ch-image' }}
	run: \|
	set -x
	[[ $(bin/ch-image gestalt python-path) = /usr/bin/python3 ]]
	[[ $(bin/ch-image -v --dependencies) = "lark path: /home/runner/work/charliecloud/charliecloud/lib/lark/__init__.py" ]]
	[[ $($ch_prefix/from-git/bin/ch-image gestalt python-path) = /usr/bin/python3 ]]
	[[ $($ch_prefix/from-git/bin/ch-image -v --dependencies) = "lark path: /var/tmp/from-git_lib/charliecloud/lark/__init__.py" ]]

	- name: late setup & validation, all
	run: \|
	bin/ch-test --is-pedantic all
	bin/ch-test --is-sudo all

	- name: make filesystem permissions fixtures
	run: \|
	bin/ch-test mk-perm-dirs

	- name: start local registry, ch-image
	if: ${{ matrix.builder == 'ch-image' }}
	# See HOWTO in Google Docs for details.
	run: \|
	set -x
	mkdir ~/registry-etc
	cp test/registry-config.yml ~/registry-etc/config.yml
	cd ~/registry-etc
	openssl req -batch -subj '/C=US/ST=NM/L=LA/O=LANL/CN=localhost' \
	-newkey rsa:4096 -nodes -sha256 -x509 -days 36500 \
	-keyout localhost.key -out localhost.crt
	#openssl x509 -text -noout -in localhost.crt
	sudo apt-get install apache2-utils
	htpasswd -Bbn charlie test > ./htpasswd
	diff -u <(docker run --rm registry:2 \
	cat /etc/docker/registry/config.yml) \
	config.yml \|\| true
	docker run -d --rm -p 127.0.0.1:5000:5000 \
	-v ~/registry-etc:/etc/docker/registry registry:2
	docker ps -a

	- name: build/install from tarball
	run: \|
	# Create and unpack tarball. The wildcard saves us having to put the
	# version in a variable. This assumes there isn’t already a tarball
	# or unpacked directory in $ch_prefix, which is true on the clean
	# VMs GitHub gives us. Note that cd fails if it gets more than one
	# argument, which helps, but this is probably kind of brittle.
	make $ch_makej dist
	mv charliecloud-*.tar.gz $ch_prefix
	cd $ch_prefix
	tar xf charliecloud-*.tar.gz
	rm charliecloud-*.tar.gz # else cd fails with “too many arguments”
	cd charliecloud-*
	pwd
	# Configure and verify output.
	./configure --prefix=$ch_prefix/from-tarball
	set -x
	fgrep 'documentation: yes' config.log
	[[ $CH_TEST_BUILDER = buildah* ]] && command -v buildah
	[[ $CH_TEST_BUILDER = docker ]] && command -v docker
	[[ $CH_TEST_BUILDER = ch-image ]] && fgrep 'with ch-image(1): yes' config.log
	fgrep 'recommended tests, tar-unpack mode: yes' config.log
	fgrep 'recommended tests, squash-unpack mode: yes' config.log
	if [[ ${{ matrix.pack_fmt }} = squash-mount ]]; then
	fgrep 'recommended tests, squash-mount mode: yes' config.log
	fgrep 'internal SquashFS mounting ... yes' config.log
	else
	fgrep 'recommended tests, squash-mount mode: no' config.log
	fgrep 'internal SquashFS mounting ... no' config.log
	fi
	set +x
	# Build and install.
	make $ch_makej
	sudo make $ch_makej install
	bin/ch-run --version
	$ch_prefix/from-tarball/bin/ch-run --version

	- name: configure sudo to user root, group non-root
	if: ${{ matrix.keep_sudo }}
	run: \|
	sudo sed -Ei 's/=$ALL$/=(ALL:ALL)/g' /etc/sudoers.d/runner
	sudo cat /etc/sudoers.d/runner

	- name: remove sudo
	if: ${{ ! matrix.keep_sudo }}
	run: \|
	sudo rm /etc/sudoers.d/runner
	! sudo echo hello

	- name: run test suite (Git WD, standard)
	run: \|
	bin/ch-test all
	# Validate that “rootemu” test phase didn’t run (skipped by default
	# on standard scope).
	[[ $(cat /tmp/ch-test.tmp.$USER/rootemu) = no ]]

	- name: run root emulation phase
	if: ${{ matrix.builder == 'ch-image'
	&& matrix.pack_fmt == 'squash-mount'
	&& matrix.cache == 'enabled' }}
	run: \|
	# We only really need to run the root emulation tests once in CI.
	# ch-image with an enabled cache is required for these unit tests.
	# Since all CI tests that use ch-image with an enabled cache take
	# roughly the same amount of time, we arbitrarily chose squash-mount
	# as the pack format.
	bin/ch-test rootemu
	# Validate that “rootemu” test phase ran.
	[[ $(cat /tmp/ch-test.tmp.$USER/rootemu) = yes ]]

	- name: run test suite (installed from Git WD, standard)
	if: ${{ matrix.builder == 'ch-image'
	&& matrix.pack_fmt == 'squash-mount'
	&& matrix.cache == 'enabled' }}
	run: \|
	$ch_prefix/from-git/bin/ch-test all
	[[ $(cat /tmp/ch-test.tmp.$USER/rootemu) = no ]]

	- name: run test suite (installed from tarball, standard)
	if: ${{ matrix.builder == 'ch-image'
	&& matrix.pack_fmt == 'squash-mount'
	&& matrix.cache == 'enabled' }}
	run: \|
	$ch_prefix/from-tarball/bin/ch-test all
	[[ $(cat /tmp/ch-test.tmp.$USER/rootemu) = no ]]

	- name: uninstall from tarball
	if: ${{ matrix.keep_sudo }}
	run: \|
	cd $ch_prefix/charliecloud-*
	sudo make uninstall
	diff -u - <(find $ch_prefix/from-tarball \| sort) <<'EOF'
	/var/tmp/from-tarball
	/var/tmp/from-tarball/bin
	/var/tmp/from-tarball/lib
	/var/tmp/from-tarball/libexec
	/var/tmp/from-tarball/share
	/var/tmp/from-tarball/share/doc
	/var/tmp/from-tarball/share/man
	/var/tmp/from-tarball/share/man/man1
	/var/tmp/from-tarball/share/man/man7
	EOF

	- name: rebuild with most things --disable’d
	if: ${{ matrix.builder == 'docker' }}
	run: \|
	make distclean
	./configure --prefix=/doesnotexist \
	--disable-html --disable-man --disable-ch-image
	set -x
	fgrep 'HTML documentation ... no' config.log
	fgrep 'man pages ... no' config.log
	fgrep 'ch-image(1) ... no' config.log
	fgrep 'with ch-image(1): no' config.log
	# This CI test requires Docker, but the “configure” script doesn’t
	# check for Docker, so we do it ourselves with “command -v”.
	command -v docker
	fgrep 'more complete tests: no' config.log
	set +x
	# Build.
	make $ch_makej
	bin/ch-run --version

	- name: run test suite (Git WD, standard)
	if: ${{ matrix.builder == 'docker' }}
	run: \|
	bin/ch-test all
	[[ $(cat /tmp/ch-test.tmp.$USER/rootemu) = no ]]

	- name: remove non-essential dependencies
	if: ${{ matrix.builder == 'docker' }}
	run: \|
	set -x
	# This breaks lots of dependencies unrelated to our build but YOLO.
	sudo dpkg --remove --force-depends \
	pigz \
	pv \
	python3-requests \
	squashfs-tools
	sudo pip3 uninstall -y sphinx sphinx-rtd-theme
	if [[ ${{ matrix.pack_fmt }} = squash-mount ]]; then
	( cd /usr/local/src/squashfuse && sudo make uninstall )
	fi
	! python3 -c 'import requests'
	! python3 -c 'import lark'

	- name: rebuild
	if: ${{ matrix.builder == 'docker' }}
	run: \|
	make distclean
	./configure --prefix=/doesnotexist
	set -x
	fgrep 'documentation: no' config.log
	command -v docker
	fgrep 'more complete tests: no' config.log
	fgrep 'recommended tests, squash-mount mode: no' config.log
	set +x
	# Build and install.
	make $ch_makej
	bin/ch-run --version

	- name: run test suite (Git WD, standard)
	if: ${{ matrix.builder == 'docker' }}
	run: \|
	bin/ch-test all
	[[ $(cat /tmp/ch-test.tmp.$USER/rootemu) = no ]]

	- name: print ending environment
	if: ${{ always() }}
	run: \|
	free -m
	df -h
	du -sch $CH_TEST_TARDIR/* \|\| true
	du -sch $CH_TEST_IMGDIR/* \|\| true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

several fixes related to `-W` testing #3000

Workflow file

several fixes related to `-W` testing #3000

Jobs

Run details

Workflow file for this run

several fixes related to -W testing #3000

Workflow file

Workflow file for this run

several fixes related to `-W` testing #3000