From 4faf6fda991e14fea9b93f84732777f0994a6055 Mon Sep 17 00:00:00 2001 From: maximsmol <1472826+maximsmol@users.noreply.github.com> Date: Fri, 23 Aug 2024 11:53:23 -0700 Subject: [PATCH 1/4] upgrade ubuntu + enable swap Signed-off-by: maximsmol <1472826+maximsmol@users.noreply.github.com> --- sysbox-eks.pkr.hcl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index 81d932d..07764bf 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -1,6 +1,6 @@ variable "ubuntu_version" { - default = "focal-20.04" + default = "noble-24.04" validation { condition = can(regex("^\\w+-\\d+\\.\\d+$", var.ubuntu_version)) @@ -211,7 +211,7 @@ build { "echo '>>> CRI-O'", # fixme(maximsmol): take into account ${ubuntu_version} - "export OS='xUbuntu_20.04'", + "export OS='xUbuntu_24.04'", "export VERSION='${var.k8s_version}'", "echo Adding repositories", @@ -434,4 +434,15 @@ build { ] } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Patching kubelet config'", + "sudo dasel put bool --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'kubeletconfig.failSwapOn' false", + "sudo dasel put bool --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'kubeletconfig.featureGates.NodeSwap' true", + ] + } } From 30cf0902ad40b8676f813d46861ed39c92aa77c6 Mon Sep 17 00:00:00 2001 From: maximsmol <1472826+maximsmol@users.noreply.github.com> Date: Fri, 23 Aug 2024 12:04:23 -0700 Subject: [PATCH 2/4] downgrade ubuntu, enable cgroups2 manually instead Signed-off-by: maximsmol <1472826+maximsmol@users.noreply.github.com> --- sysbox-eks.pkr.hcl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index 07764bf..2495b04 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -1,6 +1,6 @@ variable "ubuntu_version" { - default = "noble-24.04" + default = "focal-20.04" validation { condition = can(regex("^\\w+-\\d+\\.\\d+$", var.ubuntu_version)) @@ -211,7 +211,7 @@ build { "echo '>>> CRI-O'", # fixme(maximsmol): take into account ${ubuntu_version} - "export OS='xUbuntu_24.04'", + "export OS='xUbuntu_20.04'", "export VERSION='${var.k8s_version}'", "echo Adding repositories", @@ -434,7 +434,6 @@ build { ] } - provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ @@ -445,4 +444,14 @@ build { "sudo dasel put bool --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'kubeletconfig.featureGates.NodeSwap' true", ] } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Use cgroup2'", + "sudo grubby --update-kernel=/boot/vmlinuz-$(uname -r) --args='systemd.unified_cgroup_hierarchy=1'", + ] + } } From 054e8d3b4bfaf9e8b4b38d900b2e7b94a7da66c5 Mon Sep 17 00:00:00 2001 From: maximsmol <1472826+maximsmol@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:28:19 -0700 Subject: [PATCH 3/4] fix grub config Signed-off-by: maximsmol <1472826+maximsmol@users.noreply.github.com> --- sysbox-eks.pkr.hcl | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index 2495b04..f0b8b01 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -99,6 +99,22 @@ build { "source.amazon-ebs.ubuntu-eks", ] + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Use cgroup2'", + "sudo sed --in-place 's/GRUB_CMDLINE_LINUX_DEFAULT=\"console=tty1 console=ttyS0 nvme_core.io_timeout=4294967295\"/GRUB_CMDLINE_LINUX_DEFAULT=\"console=tty1 console=ttyS0 nvme_core.io_timeout=4294967295 systemd.unified_cgroup_hierarchy=1\"/g' /etc/default/grub.d/50-cloudimg-settings.cfg", + "sudo update-grub", + "sudo systemctl reboot" + ] + + expect_disconnect = true + skip_clean = true + pause_after = "10s" + } + provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ @@ -299,6 +315,7 @@ build { } provisioner "file" { + # reference: https://github.com/awslabs/amazon-eks-ami/blob/main/templates/al2/runtime/bootstrap.sh source = "bootstrap.sh.patch" destination = "/home/ubuntu/bootstrap.sh.patch" } @@ -444,14 +461,4 @@ build { "sudo dasel put bool --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'kubeletconfig.featureGates.NodeSwap' true", ] } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - "echo '>>> Use cgroup2'", - "sudo grubby --update-kernel=/boot/vmlinuz-$(uname -r) --args='systemd.unified_cgroup_hierarchy=1'", - ] - } } From 69cb9f695cff752300ea34e6dbd396d4e525033f Mon Sep 17 00:00:00 2001 From: maximsmol <1472826+maximsmol@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:49:44 -0700 Subject: [PATCH 4/4] fix kubelet settings Signed-off-by: maximsmol <1472826+maximsmol@users.noreply.github.com> --- sysbox-eks.pkr.hcl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index f0b8b01..5223c85 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -457,8 +457,9 @@ build { "set -o pipefail -o errexit", "echo '>>> Patching kubelet config'", - "sudo dasel put bool --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'kubeletconfig.failSwapOn' false", - "sudo dasel put bool --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'kubeletconfig.featureGates.NodeSwap' true", + "sudo dasel put bool --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'failSwapOn' false", + "sudo dasel put bool --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'featureGates.NodeSwap' true", + "sudo dasel put string --parser json --file /etc/kubernetes/kubelet/kubelet-config.json --selector 'memorySwap.swapBehavior' 'UnlimitedSwap'", ] } }