Merge pull request #11 from latchbio/taras/fuse-fix

Taras/fuse fix
latchbio · Feb 22, 2024 · a066bd3 · a066bd3
2 parents a308c4d + af56f9a
commit a066bd3
Show file tree

Hide file tree

Showing 3 changed files with 157 additions and 7 deletions.
diff --git a/Justfile b/Justfile
@@ -14,3 +14,10 @@
 
 @build:
   packer build sysbox-eks.pkr.hcl
+
+@build-crio:
+  docker build -t sysbox-eks-ami-crio . -f crio.Dockerfile
+  docker run \
+    --mount type=bind,source="$(realpath .)",target=/mnt \
+    sysbox-eks-ami-crio \
+    /usr/bin/env bash -c 'cp cri-o/bin/crio /mnt'
diff --git a/crio.Dockerfile b/crio.Dockerfile
@@ -0,0 +1,64 @@
+# syntax = docker/dockerfile:1.4.1
+
+from ubuntu:22.04 as base
+
+workdir /tmp/docker-build/work/
+
+shell [ \
+  "/usr/bin/env", "bash", \
+  "-o", "errexit", \
+  "-o", "pipefail", \
+  "-o", "nounset", \
+  "-o", "verbose", \
+  "-o", "errtrace", \
+  "-O", "inherit_errexit", \
+  "-O", "shift_verbose", \
+  "-c" \
+]
+
+env TZ='Etc/UTC'
+env LANG='en_US.UTF-8'
+
+arg DEBIAN_FRONTEND=noninteractive
+
+run --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+<<DKR
+  apt-get update
+  apt-get install \
+    --yes \
+    --no-install-recommends \
+    gnupg \
+    software-properties-common
+
+  add-apt-repository --yes \
+    ppa:longsleep/golang-backports
+DKR
+
+run --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+<<DKR
+  apt-get update
+  # todo(maximsmol): lock the golang version
+  apt-get install \
+    --yes \
+    --no-install-recommends \
+    git \
+    build-essential \
+    golang-go \
+    libgpgme-dev
+DKR
+
+run --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+<<DKR
+  git clone \
+    --branch v1.22-sysbox \
+    --depth 1 \
+    --shallow-submodules \
+    https://github.com/nestybox/cri-o.git \
+    cri-o
+
+  cd cri-o
+  make binaries
+DKR
diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl
@@ -34,12 +34,31 @@ packer {
       version = "= 1.0.9"
       source  = "github.com/hashicorp/amazon"
     }
+    git = {
+      version = ">= 0.5.0"
+      source  = "github.com/ethanmdavidson/git"
+    }
+
   }
 }
 
+data "git-commit" "current" {}
+
+local "git_branch" {
+  expression = "${substr(data.git-commit.current.hash, 0, 8)}-${replace(element(data.git-commit.current.branches, 0), "/", "-")}"
+}
+
+local "timestamp" {
+  expression = regex_replace(timestamp(), "[- TZ:]", "")
+}
+
+local "ami_name" {
+  expression = "latch-bio/sysbox-eks_${var.sysbox_version}-gpu/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-serve-${local.timestamp}-${local.git_branch}"
+}
+
 source "amazon-ebs" "ubuntu-eks" {
-  ami_name        = "latch-bio/sysbox-eks_${var.sysbox_version}/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server"
-  ami_description = "Latch Bio, Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version}, amd64 image"
+  ami_name        = "${local.ami_name}"
+  ami_description = "Latch Bio, Sysbox EKS Node (k8s_${var.k8s_version}) with NVIDIA GPU support, on Ubuntu ${var.ubuntu_version}, amd64 image."
 
   tags = {
     Linux         = "Ubuntu"
@@ -71,8 +90,7 @@ source "amazon-ebs" "ubuntu-eks" {
 build {
   name = "sysbox-eks"
   sources = [
-    "source.amazon-ebs.ubuntu-eks"
-
+    "source.amazon-ebs.ubuntu-eks",
   ]
 
   provisioner "shell" {
@@ -188,11 +206,14 @@ build {
 
   # provisioner "shell" {
   #   inline = [
-  #     "echo >>> Installing prebuilt patched CRI-O",
+  #     "echo '>>> Installing prebuilt patched CRI-O'",
   #     "sudo mv crio /usr/bin/crio",
-  #
+
   #     "echo Setting permissions",
   #     "sudo chmod u+x /usr/bin/crio"
+
+  #     # "echo Restarting CRI-O",
+  #     # "sudo systemctl restart crio"
   #   ]
   # }
 
@@ -261,7 +282,7 @@ build {
       # todo(maximsmol): do this only when K8s is configured without systemd cgroups (from sysbox todos)
       "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.cgroup_manager' 'cgroupfs'",
       "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.conmon_cgroup' 'pod'",
-      #
+
       "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETFCAP",
       "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple AUDIT_WRITE",
       "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple NET_RAW",
@@ -292,6 +313,7 @@ build {
       "echo Adding Sysbox to CRI-O runtimes",
       "sudo dasel put object --parser toml --selector 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'",
       "sudo dasel put string --parser toml --selector 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'",
+      "sudo dasel put string --parser toml --selector 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[1]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.Devices'",
     ]
   }
 
@@ -304,4 +326,61 @@ build {
       "sudo rm -r /etc/cni/net.d/",
     ]
   }
+
+  provisioner "shell" {
+    inline_shebang = "/usr/bin/env bash"
+    inline = [
+      "set -o pipefail -o errexit",
+      "export DEBIAN_FRONTEND=noninteractive",
+
+      "wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb",
+      "sudo dpkg -i cuda-keyring_1.0-1_all.deb",
+      "rm cuda-keyring_1.0-1_all.deb",
+
+      "sudo apt-get update",
+      "sudo --preserve-env=DEBIAN_FRONTEND apt-get --yes --no-install-recommends install nvidia-driver-530 nvidia-container-toolkit",
+
+      # enable mounting FUSE device inside of containers
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/fuse",
+
+      # enable mounting NVIDIA devices inside of containers
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/card0",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/card1",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/card2",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/card3",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/card4",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/card5",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/card6",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/card7",
+
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/renderD128",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/renderD129",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/renderD130",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/renderD131",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/renderD132",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/renderD133",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/renderD134",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/dri/renderD135",
+
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia0",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia1",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia2",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia3",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia4",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia5",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia6",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia7",
+
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidiactl",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia-modeset",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia-uvm",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/nvidia-uvm-tools",
+      "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.allowed_devices.[]' --multiple /dev/vga_arbiter",
+
+      "sudo dasel put string --parser toml --selector 'crio.runtime.default_runtime' --file /etc/crio/crio.conf 'nvidia'",
+      "sudo dasel put object --parser toml --selector 'crio.runtime.runtimes.nvidia' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/nvidia-container-runtime'",
+      "sudo dasel delete --parser toml --selector 'nvidia-container-runtime.runtimes' --file /etc/nvidia-container-runtime/config.toml",
+      "sudo dasel put string --parser toml --selector 'nvidia-container-runtime.runtimes.[]' --file /etc/nvidia-container-runtime/config.toml 'runc'"
+    ]
+  }
 }