From f800d5531c7a45d01f5021847b6eab6d4641a69e Mon Sep 17 00:00:00 2001 From: Mathew Gordon Date: Sun, 19 Nov 2023 05:09:10 -0700 Subject: [PATCH] Swap portable_squashfuse for squashfuse-zig, clean up build script a bit squashfuse-zig is a project I maintain and have been able to invest the time to get good static builds, along with builds targeting very old versions of glibc due to using Zig. Performance should be very comparable, besides with zlib, where a large performance gain should be expected due to squashfuse-zig using libdeflate --- .github/workflows/build.yml | 15 ++- make_runtime.sh | 178 ++++++------------------------------ runtime.sh | 2 +- 3 files changed, 37 insertions(+), 158 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f11407..ebb6338 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,9 +13,6 @@ jobs: - name: Install deps run: sudo apt install zopfli - - name: Build x86_64 (all compression, DwarFS) - run: COMPRESS_SQUASHFUSE=1 img_type=dwarfs ARCH=x86_64 ./make_runtime.sh - - name: Build x86_64 (LZ4) run: COMPRESS_SQUASHFUSE=1 ARCH=x86_64 ./make_runtime.sh @@ -55,16 +52,16 @@ jobs: # FULLY STATIC BUILDS - name: Build x86_64 (LZ4) - run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=x86_64 STATIC=-static ./make_runtime.sh + run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=x86_64 STATIC_SQUASHFUSE=1 ./make_runtime.sh - name: Build aarch64 (LZ4) - run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=aarch64 STATIC=-static ./make_runtime.sh + run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=aarch64 STATIC_SQUASHFUSE=1 ./make_runtime.sh # - name: Build armhf (LZ4) # run: COMPRESS_SQUASHFUSE=1 ARCH=armhf STATIC=-static ./make_runtime.sh - name: Build x86_64 and aarch64 (LZ4) - run: COMPRESS_SQUASHFUSE=1 ARCH=x86_64-aarch64 STATIC=-static ./make_runtime.sh + run: COMPRESS_SQUASHFUSE=1 ARCH=x86_64-aarch64 STATIC_SQUASHFUSE=1 ./make_runtime.sh # - name: Build all architectures static (LZ4) # run: COMPRESS_SQUASHFUSE=1 ARCH=x86_64-aarch64-armhf STATIC=-static ./make_runtime.sh @@ -73,13 +70,13 @@ jobs: - name: Build x86_64 (ZSTD) - run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=x86_64 COMP=zstd STATIC=-static ./make_runtime.sh + run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=x86_64 COMP=zstd STATIC_SQUASHFUSE=1 ./make_runtime.sh - name: Build aarch64 (ZSTD) - run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=aarch64 COMP=zstd STATIC=-static ./make_runtime.sh + run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=aarch64 COMP=zstd STATIC_SQUASHFUSE=1 ./make_runtime.sh - name: Build x86_64 and aarch64 (ZSTD) - run: COMPRESS_SQUASHFUSE=1 ARCH=x86_64-aarch64 COMP=zstd STATIC=-static ./make_runtime.sh + run: COMPRESS_SQUASHFUSE=1 ARCH=x86_64-aarch64 COMP=zstd STATIC_SQUASHFUSE=1 ./make_runtime.sh # - name: Build all architectures (ZSTD) # run: rm -r squashfuse; COMPRESS_SQUASHFUSE=1 ARCH=x86_64-aarch64-armhf COMP=zstd STATIC=-static ./make_runtime.sh diff --git a/make_runtime.sh b/make_runtime.sh index 1bc8201..30fca47 100755 --- a/make_runtime.sh +++ b/make_runtime.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # VERY hacked together script just to assemble the runtime, probably will # eventually make it cleaner, but it works for the time being @@ -10,58 +10,39 @@ [ -z $COMP ] && COMP='lz4' [ -z $img_type ] && img_type='squashfs' -# Make all builds static for time being -STATIC='-static' +[ $STATIC_SQUASHFUSE ] && static_prefix='.static' + +squashfuse_source="https://github.com/mgord9518/squashfuse-zig/releases/download/continuous" [ ! -d 'squashfuse' ] && mkdir squashfuse -# Download required squashfuse squashfusearies per architecture if they don't already -# exist -if [[ "$ARCH" = *'x86_64'* ]]; then - if [ ! -f 'squashfuse/squashfuse.x86_64'* ]; then - if [ $img_type = dwarfs ]; then - wget "https://github.com/mhx/dwarfs/releases/download/v0.5.6/dwarfs-0.5.6-Linux.tar.xz" -O - | \ - tar -xOJ 'dwarfs-0.5.6-Linux/sbin/dwarfs' --strip=2 > squashfuse/squashfuse.x86_64 - else - wget "https://github.com/mgord9518/portable_squashfuse/releases/download/nightly/squashfuse_ll_$COMP$STATIC.x86_64" \ - -O squashfuse/squashfuse.x86_64 - fi - fi - if [ $COMPRESS_SQUASHFUSE ]; then - zopfli --i100 squashfuse/squashfuse.x86_64 - rm squashfuse/squashfuse.x86_64 - binList="squashfuse/squashfuse.x86_64.gz" - else - binList="squashfuse/squashfuse.x86_64" - fi -fi -if [[ "$ARCH" = *'aarch64'* ]]; then - if [ ! -f 'squashfuse/squashfuse.aarch64'* ]; then - wget "https://github.com/mgord9518/portable_squashfuse/releases/download/nightly/squashfuse_ll_$COMP$STATIC.aarch64" \ - -O squashfuse/squashfuse.aarch64 - fi - if [ $COMPRESS_SQUASHFUSE ]; then - zopfli --i1000 squashfuse/squashfuse.aarch64 - rm squashfuse/squashfuse.aarch64 - binList="$binList squashfuse/squashfuse.aarch64.gz" - else - binList="$binList squashfuse/squashfuse.aarch64" - fi -fi -if [[ "$ARCH" = *'armhf'* ]]; then - if [ ! -f 'squashfuse/squashfuse.armhf'* ]; then - wget "https://github.com/mgord9518/portable_squashfuse/releases/download/manual/squashfuse_ll_$COMP$STATIC.armv7l" \ - -O squashfuse/squashfuse.armhf - fi - if [ $COMPRESS_SQUASHFUSE ]; then - zopfli --i1000 squashfuse/squashfuse.armhf - rm squashfuse/squashfuse.armhf - binList="$binList squashfuse/squashfuse.armhf.gz" - else - binList="$binList squashfuse/squashfuse.armhf" - fi +if command -v zopfli > /dev/null; then + compress_command=zopfli + compress_flags="--i100" +else + compress_command=gzip + compress_flags="-9" fi +for arch in 'x86_64' 'aarch64' 'x86' 'armhf'; do + # Download required squashfuse squashfusearies per architecture if they don't already + # exist + if [ $(grep "$arch" <<< "$ARCH") ]; then + if [ ! -f "squashfuse/squashfuse.$arch"* ]; then + wget "$squashfuse_source/squashfuse_${COMP}${static_prefix}.$arch" \ + -O "squashfuse/squashfuse.$arch" + fi + + if [ $COMPRESS_SQUASHFUSE ]; then + "$compress_command" $compress_flags "squashfuse/squashfuse.$arch" + rm "squashfuse/squashfuse.$arch" + binList="$binList squashfuse/squashfuse.$arch.gz" + else + binList="$binList squashfuse/squashfuse.$arch" + fi + fi +done + # Collapse the script to make it smaller, not really sure whether I should keep # it or not as it also obfuscates the code and the size difference makes little # difference as the squashfuse binaries make up an overwhelming majority of the @@ -70,106 +51,7 @@ echo '#!/bin/sh #.shImg.# #see for src' > runtime -# Experimental e x t r a flattening, might turn this into its own seperate -# project -# this has really gotten out of hand -if [ $COMPRESS_SCRIPT ]; then - echo 'alias A=alias -A B=else -A C=cut -A D="sed -e" -A E=echo -A F="command -v" -A G=grep -A H=head -A L=test -A I="if L" -A J="elif L" -A K=gzip -A M=mkdir -A N=then -A O="L -z" -A P="L -f" -A Q="case" -A R="1>&2 E" -A S=sed -A T=tail -A U="E -ne" -A V=wait -A W="K -d" -A X=fi -A Y=exit -A Z=return -A A="xxd -s"' >> runtime -# ^ alias common commands and statements to single chars to drastically shrink -# scripts - - # Replace common shell phrases with aliases - cat runtime.sh | tr -d '\t' | sed \ - -e 's/#.*//' \ - -e 's/ \&\& /\&\&/g' \ - -e 's/ \&/\&/g' \ - -e 's/ || /||/g' \ - -e 's/ | /|/g' \ - -e 's/ {/{/g' \ - -e 's/; /;/g' \ - -e 's/ > />/g' \ - -e 's/> />/g' \ - -e 's/ < /&2 echo/R/g' \ - -e 's/^echo -ne/U/g' \ - -e 's/^echo -en/U/g' \ - -e 's/^echo -e -n/U/g' \ - -e 's/^echo -n -e/U/g' \ - -e 's/^echo/E/g' \ - -e 's/\&\&echo -ne/\&\&U/g' \ - -e 's/\&\&echo -en/\&\&U/g' \ - -e 's/\&\&echo -e -n/\&\&U/g' \ - -e 's/\&\&echo -n -e/\&\&U/g' \ - -e 's/\&\&echo/\&\&E/g' \ - -e 's/^exit/Y/g' \ - -e 's/mkdir/M/g' \ - -e 's/sed -e/D/g' \ - -e 's/^sed/S/g' \ - -e 's/|sed/|S/g' \ - -e 's/xxd -s/A/g' \ - -e 's/xxd -e -s/A -e/g' \ - -e 's/command -v/F/g' \ - -e 's/gzip -d/W/g' \ - -e 's/gunzip/W/g' \ - -e 's/wait/V/g' \ - -e 's/gzip/K/g' \ - -e 's/grep/G/g' \ - -e 's/^elif \[/J/g' \ - -e 's/^if \[/I/g' \ - -e 's/^then/N/g' \ - -e 's/;then/;N/g' \ - -e 's/^else/B/g' \ - -e 's/^fi/X/g' \ - -e 's/^return/Z/g' \ - -e 's/\&\&return/\&\&Z/g' \ - -e 's/\[ /L /g' \ - -e 's/L -z/O/g' \ - -e 's/L -f/P/g' \ - -e 's/case/Q/g' \ - -e 's/ \]//g' \ - | perl -0pe 's/;;\nesac/;;esac/g' | grep . >> runtime -else - cat runtime.sh | tr -d '\t' | sed 's/#.*//' | grep . >> runtime -fi +cat runtime.sh | tr -d '\t' | sed 's/#.*//' | grep . >> runtime arch=$(echo "$ARCH" | tr '-' ';') diff --git a/runtime.sh b/runtime.sh index 338968e..c23f014 100644 --- a/runtime.sh +++ b/runtime.sh @@ -55,7 +55,7 @@ image_offset=_IMAGE_OFFSET_ # This will get switched to `y` if `add_integration.sh` is run contains_integration_zip=n -version=0.3.0 +version=0.4.0 arch=_ARCH_ # Run these startup commands concurrently to make them faster