Merge branch 'main' into network-worker

.github/workflows/docs.yml

@@ -8,7 +8,7 @@ on:
     paths:
       - 'docs/**'
       - 'mkdocs.yml'
-      - '.github/workflows/**'
+      - '.github/workflows/docs.yml'
 
 defaults:
   run:

.github/workflows/kernel.yml

@@ -0,0 +1,47 @@
+name: Kernel binary build
+
+on:
+  push:
+    branches:
+      - main
+
+    paths:
+      - 'src/**'
+      - 'ext/**'
+      - 'Makefile.static'
+      - 'configure'
+      - '.github/workflows/kernel.yml'
+
+jobs:
+  kernel:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/lutoma/xelix/toolchain:latest
+
+    steps:
+      - name: Download PKGBUILD
+        run: 'wget -q https://raw.githubusercontent.com/lutoma/xelix/main/land/xelix-kernel/PKGBUILD'
+
+      - name: Adjust dir permissions
+        run: 'sudo chown -R dev: .'
+
+      - name: Import signing key
+        run: 'echo "${{ secrets.GPG_PRIVATE_KEY }}" | sudo -u dev gpg --import'
+
+      - name: Run makepkg
+        run:  'sudo -u dev makepkg -Ad --sign'
+
+      - name: Adjust package name
+        run: 'rename x86_64 i786 *'
+
+      - name: 'Create package directory'
+        run: 'mkdir kernel-pkg'
+
+      - name: Import SSH key
+        run: 'echo "${{ secrets.SSH_KEY }}" > /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519'
+
+      - name: Upload to repo server
+        run: rsync *.pkg.tar.gz* repoupdate@pkgs.xelix.org:/var/www/pkgs.xelix.org/core/i786/
+
+      - name: Update repository
+        run: ssh repoupdate@pkgs.xelix.org /usr/local/bin/repo-remote-update core i786 xelix-kernel *.pkg.tar.gz

.github/workflows/nightly.yaml

@@ -0,0 +1,44 @@
+name: Build nightly image
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '35 4 * * *'
+
+jobs:
+  image:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/lutoma/xelix/toolchain:latest
+      options: --privileged
+
+    steps:
+      - name: Download build script
+        run: 'wget -q https://raw.githubusercontent.com/lutoma/xelix/main/util/build-image.sh'
+
+      - name: Build image
+        run: bash build-image.sh xelix-nightly.img
+
+      - name: Convert image
+        run: |
+          qemu-img convert -f raw xelix-nightly.img -O qcow2 xelix-nightly.qcow2
+          qemu-img convert -f raw xelix-nightly.img -O vdi xelix-nightly.vdi
+          qemu-img convert -f raw xelix-nightly.img -O vmdk xelix-nightly.vmdk
+          qemu-img convert -f raw xelix-nightly.img -O vhdx xelix-nightly.vhdx
+
+      - name: Import signing key
+        run: 'echo "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --import'
+
+      - name: Sign images
+        run: |
+          gpg --detach-sign --armor xelix-nightly.qcow2
+          gpg --detach-sign --armor xelix-nightly.vdi
+          gpg --detach-sign --armor xelix-nightly.vmdk
+          gpg --detach-sign --armor xelix-nightly.vhdx
+
+      - name: Import SSH key
+        run: 'echo "${{ secrets.SSH_KEY }}" > /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519'
+
+
+      - name: Upload to server
+        run: rsync -z --exclude xelix-nightly.img xelix-nightly.* repoupdate@pkgs.xelix.org:/var/www/pkgs.xelix.org

.github/workflows/toolchain.yml

@@ -0,0 +1,69 @@
+name: Toolchain container build
+
+on:
+  push:
+    branches:
+      - main
+
+    paths:
+      - 'land/binutils/**'
+      - 'land/gcc/**'
+      - 'land/newlib/**'
+      - 'Containerfile'
+      - '.github/workflows/toolchain.yml'
+
+jobs:
+  toolchain:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out git repository
+        uses: actions/checkout@v4
+
+      - name: Prepare environment
+        id: prep
+        run: |
+          TAG=$(echo $GITHUB_SHA | head -c7)
+          IMAGE="ghcr.io/${{ github.repository }}/toolchain"
+          echo ::set-output name=tagged_image::${IMAGE}:${TAG}
+          echo ::set-output name=tag::${TAG}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx
+
+      - name: Building image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          builder: ${{ steps.buildx.outputs.name }}
+          file: Containerfile
+          push: true
+          tags: |
+            ${{ steps.prep.outputs.tagged_image }}
+            ghcr.io/${{ github.repository }}/toolchain:latest
+
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.gitignore

@@ -14,8 +14,9 @@ tags
 src/lib/config.h
 xelix.map
 .bochsrc
-/xpkg-build
-.xpkg-status
 /site
-.config
-.config.old
+/.config
+/.config.old
+/land/*/pkg
+/land/*/src
+/land/*/*.tar.*

Containerfile

@@ -0,0 +1,148 @@
+FROM alpine:latest
+LABEL maintainer="hello@lutoma.org"
+
+ARG TARGET=i786-pc-xelix
+
+ARG BINUTILS_VERSION=2.40
+ARG BINUTILS_URL=https://ftp.gnu.org/gnu/binutils
+ARG BINUTILS_PACKAGE=binutils-${BINUTILS_VERSION}.tar.xz
+
+ARG GCC_VERSION=13.1.0
+ARG GCC_URL=https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}
+ARG GCC_PACKAGE=gcc-${GCC_VERSION}.tar.xz
+
+ARG NEWLIB_VERSION=3.2.0
+ARG NEWLIB_URL=https://sourceware.org/pub/newlib
+ARG NEWLIB_PACKAGE=newlib-${NEWLIB_VERSION}.tar.gz
+
+ENV PATH="/toolchain/usr/bin:${PATH}"
+
+RUN apk --no-cache add wget musl-dev make gcc g++ m4 perl autoconf automake \
+	patch libtool mpc1-dev gmp-dev mpfr-dev gawk texinfo file
+
+# Build outdated autoconf and automake versions for newlib
+WORKDIR /usr/src
+RUN wget https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz
+RUN wget https://ftp.gnu.org/gnu/automake/automake-1.11.6.tar.xz
+RUN tar xf autoconf-2.69.tar.gz
+RUN tar xf automake-1.11.6.tar.xz
+
+WORKDIR /usr/src/autoconf-2.69
+RUN ./configure --program-suffix=-2.69
+RUN make all install
+
+WORKDIR /usr/src/automake-1.11.6
+RUN ./configure --program-suffix=-1.11
+RUN make all install
+RUN ln -s /usr/local/share/aclocal-1.11 /usr/local/share/aclocal
+
+# Download sources and apply Xelix patches/support files
+WORKDIR /usr/src
+RUN wget -c ${BINUTILS_URL}/${BINUTILS_PACKAGE}
+RUN wget -c ${GCC_URL}/${GCC_PACKAGE}
+RUN wget -c ${NEWLIB_URL}/${NEWLIB_PACKAGE}
+
+RUN tar -xf ${BINUTILS_PACKAGE}
+RUN tar -xf ${GCC_PACKAGE}
+RUN tar -xf ${NEWLIB_PACKAGE}
+
+COPY land/binutils/binutils-${BINUTILS_VERSION}.patch /usr/src/
+RUN patch -p0 -dbinutils-${BINUTILS_VERSION}/ < binutils-${BINUTILS_VERSION}.patch
+COPY land/binutils/elf_i386_xelix.sh binutils-${BINUTILS_VERSION}/ld/emulparams/
+COPY land/binutils/elf_x86_64_xelix.sh binutils-${BINUTILS_VERSION}/ld/emulparams/
+
+RUN cd binutils-${BINUTILS_VERSION}/ld && aclocal
+RUN cd binutils-${BINUTILS_VERSION}/ld && automake
+RUN cd binutils-${BINUTILS_VERSION} && autoreconf-2.69
+
+COPY land/gcc/gcc-${GCC_VERSION}.patch /usr/src/
+RUN patch -p0 < gcc-${GCC_VERSION}.patch
+COPY land/gcc/xelix.h gcc-${GCC_VERSION}/gcc/config/
+
+COPY land/newlib/newlib-${NEWLIB_VERSION}.patch /usr/src/
+RUN patch -p0 < newlib-${NEWLIB_VERSION}.patch
+
+COPY land/newlib/xelix newlib-${NEWLIB_VERSION}/newlib/libc/sys/xelix
+
+WORKDIR /usr/src/newlib-${NEWLIB_VERSION}/newlib/libc/sys/xelix/
+RUN aclocal-1.11 -I ../../../ -I ../../../../
+RUN autoconf-2.69
+RUN automake-1.11 --cygnus Makefile
+
+WORKDIR /usr/src/newlib-${NEWLIB_VERSION}/newlib/libc/sys/
+RUN aclocal-1.11 -I ../.. -I ../../..
+RUN autoconf-2.69
+RUN automake-1.11 --cygnus Makefile
+
+WORKDIR /build/binutils
+RUN /usr/src/binutils-${BINUTILS_VERSION}/configure \
+	--target=${TARGET} \
+	--prefix=/usr \
+	--sysconfdir=/etc \
+	--disable-nls \
+	--disable-werror
+
+RUN make DESTDIR=/toolchain all install
+
+# GCC build uses newlib files, so configure that first
+WORKDIR /build/newlib
+RUN /usr/src/newlib-${NEWLIB_VERSION}/configure \
+	--target=${TARGET} \
+	--prefix=/usr \
+	--sysconfdir=/etc \
+	--enable-newlib-mb \
+	--enable-newlib-iconv \
+	--enable-newlib-io-c99-formats \
+	--enable-newlib-io-long-long \
+	--enable-newlib-io-long-double
+
+WORKDIR /build/gcc
+RUN /usr/src/gcc-${GCC_VERSION}/configure \
+	--target=${TARGET} \
+	--prefix=/usr \
+	--sysconfdir /etc \
+	--disable-nls \
+	--enable-languages=c,c++ \
+	--with-headers=/usr/src/newlib-${NEWLIB_VERSION}/newlib/libc/include \
+	--without-docdir \
+	--with-newlib
+
+RUN make -j$(nproc) all-gcc
+RUN make -j$(nproc) all-target-libgcc
+RUN make -j$(nproc) all-target-libstdc++-v3
+
+RUN make DESTDIR=/toolchain install-gcc install-target-libgcc install-target-libstdc++-v3
+RUN ln -s i786-pc-xelix-gcc /toolchain/usr/bin/i786-pc-xelix-cc
+
+WORKDIR /build/newlib
+RUN make -j$(nproc) all
+RUN make DESTDIR=/toolchain install
+RUN cp i786-pc-xelix/newlib/libc/sys/xelix/crti.o i786-pc-xelix/newlib/libc/sys/xelix/crtn.o /toolchain/usr/i786-pc-xelix/lib/
+
+# Strip debug info from binaries (Reduces image size substantially)
+RUN strip --strip-unneeded /toolchain/usr/bin/i786-pc-xelix-* /toolchain/usr/i786-pc-xelix/bin/*
+RUN find /toolchain/usr/libexec/gcc/i786-pc-xelix/13.1.0 -type f -exec strip --strip-unneeded {} \;
+
+# Now build the actual image
+FROM alpine:latest
+WORKDIR /src
+COPY --from=0 /toolchain /
+COPY --from=0 /usr/local /usr/local
+
+RUN apk --no-cache add wget git make gcc g++ nasm m4 perl autoconf automake \
+	patch libtool mpc1 gmp mpfr libarchive gettext gawk bash coreutils \
+	texinfo file python3 tar findutils gzip xz meson ninja sudo curl pacman \
+	fakeroot util-linux-misc openssh-client-default rsync nano bison flex \
+	qemu-img grub-bios sfdisk e2fsprogs moreutils bison flex pkgconfig gperf \
+	ripgrep tzdata-utils
+
+# Add non-root user for makepkg
+RUN adduser --disabled-password --gecos '' dev
+RUN echo 'dev ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+# For uploads in automated runs
+RUN mkdir /root/.ssh
+RUN ssh-keyscan pkgs.xelix.org >> /root/.ssh/known_hosts
+
+COPY land/pacman/pacman.linux.conf /etc/pacman.conf
+CMD ["/bin/bash"]

Kconfig

@@ -26,8 +26,8 @@ menu "Kernel stdlib"
 		Store kernel log in memory and make them available as /sys/log, where
 		it can be read by utilities like dmesg. Doesn't log debug messages.
 
-	config LOG_PRINT_LEVEL
-		int "TTY log level"
+	config LOG_CONSOLE_LEVEL
+		int "Default console log level"
 		default 2
 		---help---
 		0 Disabled
@@ -146,6 +146,10 @@ menu "File systems"
 	bool "Enable VirtIO block device driver"
 	default y
 
+	config ENABLE_FTREE
+	bool "Enable ftree file tracking (likely broken)"
+	default n
+
 	config ENABLE_EXT2
 	bool "Enable ext2 support"
 	default y

Makefile.static

@@ -13,13 +13,12 @@ else
 	AS = nasm
 endif
 
-PATH += :toolchain/local/bin
 CC := $(PREFIX)-gcc
 OBJCOPY := $(PREFIX)-objcopy
 QEMU := qemu-system-$(ARCH)
 LD := $(CC)
 
-MAKE_CFLAGS += -std=gnu18 -pipe -g -O3 -ffreestanding -fno-omit-frame-pointer -fstack-protector-strong -Wall -Wframe-larger-than=1024 -Wcast-align $(INCLUDE)
+MAKE_CFLAGS += -std=gnu2x -pipe -g -O3 -ffreestanding -fno-omit-frame-pointer -fstack-protector-strong -Wall -Wshadow -Wstrict-prototypes -Wmissing-prototypes -Wframe-larger-than=1024 -Wcast-align -Wduplicated-cond -Wduplicated-branches -Wlogical-op -Wnull-dereference $(INCLUDE)
 MAKE_LDFLAGS := -T src/boot/$(ARCH)-linker.ld -nostdlib -lgcc
 
 ifeq ($(shell uname -s), Darwin)

configure

@@ -25,7 +25,7 @@ function w {
 FIND=$(command -v gfind || command -v find)
 
 # Update submodules
-git submodule update --init 1> /dev/null
+[ -x "$(command -v git)" ] && git submodule update --init 1> /dev/null
 
 # Make sure config.h exists on first build
 if [ ! -f ".config" ]; then