Merge pull request #2906 from chrischdi/pr-templates-use-dedicated-di…

…r-for-kubeadm-scripts 🌱 templates: add a script to run prekubeadmcommands in order of files defined in /etc/pre-kubeadm-scripts
kubernetes-sigs · Apr 15, 2024 · c26e53d · c26e53d
2 parents 231965b + 2704ecb
commit c26e53d
Show file tree

Hide file tree

Showing 13 changed files with 61 additions and 53 deletions.
diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go
@@ -603,13 +603,19 @@ func defaultPreKubeadmCommands() []string {
 		"hostnamectl set-hostname \"{{ ds.meta_data.hostname }}\"",
 		"echo \"::1         ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6\" >/etc/hosts",
 		"echo \"127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4\" >>/etc/hosts",
+		// Ensure the directory exists so the find does not fail if no files got created.
+		"mkdir -p /etc/pre-kubeadm-commands",
+		"for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort); do echo \"Running script $script\"; \"$script\"; done",
 	}
 }
 
 func flatcarPreKubeadmCommands() []string {
 	return []string{
 		"envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp",
 		"mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml",
+		// Ensure the directory exists so the find does not fail if no files got created.
+		"mkdir -p /etc/pre-kubeadm-commands",
+		"for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort); do echo \"Running script $script\"; \"$script\"; done",
 	}
 }
 

diff --git a/packaging/flavorgen/flavors/kubevip/files.go b/packaging/flavorgen/flavors/kubevip/files.go
@@ -53,10 +53,10 @@ func newKubeVIPFiles() []bootstrapv1.File {
 			Permissions: "0644",
 			Content:     "127.0.0.1 localhost kubernetes",
 		},
-		// This two files are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684
+		// This file is part of the workaround for https://github.com/kube-vip/kube-vip/issues/684
 		{
 			Owner:       "root:root",
-			Path:        "/etc/kube-vip-prepare.sh",
+			Path:        "/etc/pre-kubeadm-commands/50-kube-vip-prepare.sh",
 			Permissions: "0700",
 			Content:     kubeVipPrepare,
 		},

diff --git a/packaging/flavorgen/flavors/kubevip/kubevip.go b/packaging/flavorgen/flavors/kubevip/kubevip.go
@@ -24,10 +24,4 @@ import (
 // PatchControlPlane adds kube-vip to a KubeadmControlPlane object.
 func PatchControlPlane(cp *controlplanev1.KubeadmControlPlane) {
 	cp.Spec.KubeadmConfigSpec.Files = append(cp.Spec.KubeadmConfigSpec.Files, newKubeVIPFiles()...)
-
-	// This commands is part of the workaround for https://github.com/kube-vip/kube-vip/issues/684
-	cp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
-		cp.Spec.KubeadmConfigSpec.PreKubeadmCommands,
-		"/etc/kube-vip-prepare.sh",
-	)
 }
diff --git a/packaging/flavorgen/flavors/kubevip/topology.go b/packaging/flavorgen/flavors/kubevip/topology.go
@@ -77,15 +77,7 @@ func TopologyPatch() clusterv1.ClusterClassPatch {
 		patches = append(patches, p)
 	}
 
-	// This two patches are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684
-	patches = append(patches,
-		clusterv1.JSONPatch{
-			Op:        "add",
-			Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-",
-			ValueFrom: &clusterv1.JSONPatchValue{Template: ptr.To("/etc/kube-vip-prepare.sh")},
-		},
-	)
-
+	// This two patches is part of the workaround for https://github.com/kube-vip/kube-vip/issues/684
 	return clusterv1.ClusterClassPatch{
 		Name: "kubeVipPodManifest",
 		Definitions: []clusterv1.PatchDefinition{

diff --git a/templates/cluster-template-external-loadbalancer.yaml b/templates/cluster-template-external-loadbalancer.yaml
@@ -121,6 +121,9 @@ spec:
       >/etc/hosts
     - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
       localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+    - mkdir -p /etc/pre-kubeadm-commands
+    - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+      do echo "Running script $script"; "$script"; done
     users:
     - name: capv
       sshAuthorizedKeys:
@@ -154,6 +157,9 @@ spec:
         >/etc/hosts
       - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
         localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+      - mkdir -p /etc/pre-kubeadm-commands
+      - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+        do echo "Running script $script"; "$script"; done
       users:
       - name: capv
         sshAuthorizedKeys:

diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml
@@ -199,7 +199,7 @@ spec:
             /etc/kubernetes/manifests/kube-vip.yaml
         fi
       owner: root:root
-      path: /etc/kube-vip-prepare.sh
+      path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh
       permissions: "0700"
     format: ignition
     ignition:
@@ -277,7 +277,9 @@ spec:
     preKubeadmCommands:
     - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
     - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
-    - /etc/kube-vip-prepare.sh
+    - mkdir -p /etc/pre-kubeadm-commands
+    - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+      do echo "Running script $script"; "$script"; done
     users:
     - name: core
       sshAuthorizedKeys:
@@ -369,6 +371,9 @@ spec:
       preKubeadmCommands:
       - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
       - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
+      - mkdir -p /etc/pre-kubeadm-commands
+      - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+        do echo "Running script $script"; "$script"; done
       users:
       - name: core
         sshAuthorizedKeys:

diff --git a/templates/cluster-template-node-ipam.yaml b/templates/cluster-template-node-ipam.yaml
@@ -236,7 +236,7 @@ spec:
             /etc/kubernetes/manifests/kube-vip.yaml
         fi
       owner: root:root
-      path: /etc/kube-vip-prepare.sh
+      path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh
       permissions: "0700"
     initConfiguration:
       nodeRegistration:
@@ -256,7 +256,9 @@ spec:
       >/etc/hosts
     - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
       localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
-    - /etc/kube-vip-prepare.sh
+    - mkdir -p /etc/pre-kubeadm-commands
+    - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+      do echo "Running script $script"; "$script"; done
     users:
     - name: capv
       sshAuthorizedKeys:
@@ -290,6 +292,9 @@ spec:
         >/etc/hosts
       - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
         localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+      - mkdir -p /etc/pre-kubeadm-commands
+      - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+        do echo "Running script $script"; "$script"; done
       users:
       - name: capv
         sshAuthorizedKeys:

diff --git a/templates/cluster-template-supervisor.yaml b/templates/cluster-template-supervisor.yaml
@@ -193,7 +193,7 @@ spec:
             /etc/kubernetes/manifests/kube-vip.yaml
         fi
       owner: root:root
-      path: /etc/kube-vip-prepare.sh
+      path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh
       permissions: "0700"
     initConfiguration:
       nodeRegistration:
@@ -214,7 +214,9 @@ spec:
       >/etc/hosts
     - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
       localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
-    - /etc/kube-vip-prepare.sh
+    - mkdir -p /etc/pre-kubeadm-commands
+    - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+      do echo "Running script $script"; "$script"; done
     users:
     - name: capv
       sshAuthorizedKeys:
@@ -249,6 +251,9 @@ spec:
         >/etc/hosts
       - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
         localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+      - mkdir -p /etc/pre-kubeadm-commands
+      - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+        do echo "Running script $script"; "$script"; done
       users:
       - name: capv
         sshAuthorizedKeys:

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
@@ -226,7 +226,7 @@ spec:
             /etc/kubernetes/manifests/kube-vip.yaml
         fi
       owner: root:root
-      path: /etc/kube-vip-prepare.sh
+      path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh
       permissions: "0700"
     initConfiguration:
       nodeRegistration:
@@ -246,7 +246,9 @@ spec:
       >/etc/hosts
     - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
       localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
-    - /etc/kube-vip-prepare.sh
+    - mkdir -p /etc/pre-kubeadm-commands
+    - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+      do echo "Running script $script"; "$script"; done
     users:
     - name: capv
       sshAuthorizedKeys:
@@ -280,6 +282,9 @@ spec:
         >/etc/hosts
       - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
         localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+      - mkdir -p /etc/pre-kubeadm-commands
+      - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+        do echo "Running script $script"; "$script"; done
       users:
       - name: capv
         sshAuthorizedKeys:

diff --git a/templates/clusterclass-template-supervisor.yaml b/templates/clusterclass-template-supervisor.yaml
@@ -174,12 +174,8 @@ spec:
                   /etc/kubernetes/manifests/kube-vip.yaml
               fi
             owner: root:root
-            path: /etc/kube-vip-prepare.sh
+            path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh
             permissions: "0700"
-      - op: add
-        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
-        valueFrom:
-          template: /etc/kube-vip-prepare.sh
       selector:
         apiVersion: controlplane.cluster.x-k8s.io/v1beta1
         kind: KubeadmControlPlaneTemplate
@@ -294,6 +290,9 @@ spec:
           >/etc/hosts
         - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
           localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+        - mkdir -p /etc/pre-kubeadm-commands
+        - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+          do echo "Running script $script"; "$script"; done
         users:
         - name: capv
           sshAuthorizedKeys:
@@ -321,3 +320,6 @@ spec:
         >/etc/hosts
       - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
         localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+      - mkdir -p /etc/pre-kubeadm-commands
+      - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+        do echo "Running script $script"; "$script"; done
diff --git a/templates/clusterclass-template.yaml b/templates/clusterclass-template.yaml
@@ -188,12 +188,8 @@ spec:
                   /etc/kubernetes/manifests/kube-vip.yaml
               fi
             owner: root:root
-            path: /etc/kube-vip-prepare.sh
+            path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh
             permissions: "0700"
-      - op: add
-        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
-        valueFrom:
-          template: /etc/kube-vip-prepare.sh
       selector:
         apiVersion: controlplane.cluster.x-k8s.io/v1beta1
         kind: KubeadmControlPlaneTemplate
@@ -353,6 +349,9 @@ spec:
           >/etc/hosts
         - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
           localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+        - mkdir -p /etc/pre-kubeadm-commands
+        - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+          do echo "Running script $script"; "$script"; done
         users:
         - name: capv
           sshAuthorizedKeys:
@@ -379,3 +378,6 @@ spec:
         >/etc/hosts
       - echo "127.0.0.1   {{ ds.meta_data.hostname }} {{ local_hostname }} localhost
         localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts
+      - mkdir -p /etc/pre-kubeadm-commands
+      - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort);
+        do echo "Running script $script"; "$script"; done
diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-prekubeadmscript.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass/patch-prekubeadmscript.yaml
@@ -3,15 +3,12 @@
   value:
     definitions:
     - jsonPatches:
-      - op: add
-        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/0
-        value: "/opt/prekubeadmscript.sh"
       - op: add
         path: /spec/template/spec/kubeadmConfigSpec/files/-
         valueFrom:
           template: |
             owner: root:root
-            path:  "/opt/prekubeadmscript.sh"
+            path:  "/etc/pre-kubeadm-commands/10-prekubeadmscript.sh"
             permissions: "0755"
             content: {{ printf "%q" (regexReplaceAll "(KUBERNETES_VERSION=.*)" .preKubeadmScript (printf "KUBERNETES_VERSION=%s" .builtin.controlPlane.version)) }}
       selector:
@@ -20,15 +17,12 @@
         matchResources:
           controlPlane: true
     - jsonPatches:
-      - op: add
-        path: /spec/template/spec/preKubeadmCommands/0
-        value: "/opt/prekubeadmscript.sh"
       - op: add
         path: /spec/template/spec/files/-
         valueFrom:
           template: |
             owner: root:root
-            path:  "/opt/prekubeadmscript.sh"
+            path:  "/etc/pre-kubeadm-commands/10-prekubeadmscript.sh"
             permissions: "0755"
             content: {{ printf "%q" (regexReplaceAll "(KUBERNETES_VERSION=.*)" .preKubeadmScript (printf "KUBERNETES_VERSION=%s" .builtin.machineDeployment.version)) }}
       selector:

diff --git a/.../e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-prekubeadmscript.yaml b/.../e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-prekubeadmscript.yaml
@@ -3,16 +3,12 @@
   value:
     definitions:
     - jsonPatches:
-      - op: add
-        # Note: We are adding prekubeadmscript.sh (install on bootstrap) at the end of the preKubeadmCommands because getting an ip for the machine (dhclient eth0) must always be executed as a first command.
-        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
-        value: "/opt/prekubeadmscript.sh"
       - op: add
         path: /spec/template/spec/kubeadmConfigSpec/files/-
         valueFrom:
           template: |
             owner: root:root
-            path:  "/opt/prekubeadmscript.sh"
+            path:  "/etc/pre-kubeadm-commands/10-prekubeadmscript.sh"
             permissions: "0755"
             content: {{ printf "%q" (regexReplaceAll "(KUBERNETES_VERSION=.*)" .preKubeadmScript (printf "KUBERNETES_VERSION=%s" .builtin.controlPlane.version)) }}
       selector:
@@ -21,16 +17,12 @@
         matchResources:
           controlPlane: true
     - jsonPatches:
-      - op: add
-        # Note: We are adding prekubeadmscript.sh (install on bootstrap) at the end of the preKubeadmCommands because getting an ip for the machine (dhclient eth0) must always be executed as a first command.
-        path: /spec/template/spec/preKubeadmCommands/-
-        value: "/opt/prekubeadmscript.sh"
       - op: add
         path: /spec/template/spec/files/-
         valueFrom:
           template: |
             owner: root:root
-            path:  "/opt/prekubeadmscript.sh"
+            path:  "/etc/pre-kubeadm-commands/10-prekubeadmscript.sh"
             permissions: "0755"
             content: {{ printf "%q" (regexReplaceAll "(KUBERNETES_VERSION=.*)" .preKubeadmScript (printf "KUBERNETES_VERSION=%s" .builtin.machineDeployment.version)) }}
       selector: