Merge pull request kubernetes-sigs#10947 from sbueringer/pr-fix-kubeadm

✨ KCP: default ControlPlaneKubeletLocalMode feature gate to true for >= 1.31.0

controlplane/kubeadm/internal/controllers/controller_test.go

@@ -2351,7 +2351,7 @@ func createClusterWithControlPlane(namespace string) (*clusterv1.Cluster, *contr
 				},
 			},
 			Replicas: ptr.To[int32](int32(3)),
-			Version:  "v1.16.6",
+			Version:  "v1.31.0",
 			RolloutStrategy: &controlplanev1.RolloutStrategy{
 				Type: "RollingUpdate",
 				RollingUpdate: &controlplanev1.RollingUpdate{

controlplane/kubeadm/internal/controllers/scale.go

@@ -33,12 +33,22 @@ import (
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal"
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/conditions"
+	"sigs.k8s.io/cluster-api/util/version"
 )
 
 func (r *KubeadmControlPlaneReconciler) initializeControlPlane(ctx context.Context, controlPlane *internal.ControlPlane) (ctrl.Result, error) {
 	logger := ctrl.LoggerFrom(ctx)
 
 	bootstrapSpec := controlPlane.InitialControlPlaneConfig()
+
+	// We intentionally only parse major/minor/patch so that the subsequent code
+	// also already applies to beta versions of new releases.
+	parsedVersionTolerant, err := version.ParseMajorMinorPatchTolerant(controlPlane.KCP.Spec.Version)
+	if err != nil {
+		return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version)
+	}
+	internal.DefaultFeatureGates(bootstrapSpec, parsedVersionTolerant)
+
 	fd, err := controlPlane.NextFailureDomainForScaleUp(ctx)
 	if err != nil {
 		return ctrl.Result{}, err
@@ -64,6 +74,15 @@ func (r *KubeadmControlPlaneReconciler) scaleUpControlPlane(ctx context.Context,
 
 	// Create the bootstrap configuration
 	bootstrapSpec := controlPlane.JoinControlPlaneConfig()
+
+	// We intentionally only parse major/minor/patch so that the subsequent code
+	// also already applies to beta versions of new releases.
+	parsedVersionTolerant, err := version.ParseMajorMinorPatchTolerant(controlPlane.KCP.Spec.Version)
+	if err != nil {
+		return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version)
+	}
+	internal.DefaultFeatureGates(bootstrapSpec, parsedVersionTolerant)
+
 	fd, err := controlPlane.NextFailureDomainForScaleUp(ctx)
 	if err != nil {
 		return ctrl.Result{}, err

controlplane/kubeadm/internal/controllers/scale_test.go

@@ -102,6 +102,11 @@ func TestKubeadmControlPlaneReconciler_initializeControlPlane(t *testing.T) {
 	g.Expect(machineList.Items[0].Spec.Bootstrap.ConfigRef.Name).To(Equal(machineList.Items[0].Name))
 	g.Expect(machineList.Items[0].Spec.Bootstrap.ConfigRef.APIVersion).To(Equal(bootstrapv1.GroupVersion.String()))
 	g.Expect(machineList.Items[0].Spec.Bootstrap.ConfigRef.Kind).To(Equal("KubeadmConfig"))
+
+	kubeadmConfig := &bootstrapv1.KubeadmConfig{}
+	bootstrapRef := machineList.Items[0].Spec.Bootstrap.ConfigRef
+	g.Expect(env.GetAPIReader().Get(ctx, client.ObjectKey{Namespace: bootstrapRef.Namespace, Name: bootstrapRef.Name}, kubeadmConfig)).To(Succeed())
+	g.Expect(kubeadmConfig.Spec.ClusterConfiguration.FeatureGates).To(BeComparableTo(map[string]bool{internal.ControlPlaneKubeletLocalMode: true}))
 }
 
 func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) {
@@ -165,6 +170,11 @@ func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) {
 		// Note: expected length is 1 because only the newly created machine is on API server. Other machines are
 		// in-memory only during the test.
 		g.Expect(controlPlaneMachines.Items).To(HaveLen(1))
+
+		kubeadmConfig := &bootstrapv1.KubeadmConfig{}
+		bootstrapRef := controlPlaneMachines.Items[0].Spec.Bootstrap.ConfigRef
+		g.Expect(env.GetAPIReader().Get(ctx, client.ObjectKey{Namespace: bootstrapRef.Namespace, Name: bootstrapRef.Name}, kubeadmConfig)).To(Succeed())
+		g.Expect(kubeadmConfig.Spec.ClusterConfiguration.FeatureGates).To(BeComparableTo(map[string]bool{internal.ControlPlaneKubeletLocalMode: true}))
 	})
 	t.Run("does not create a control plane Machine if preflight checks fail", func(t *testing.T) {
 		setup := func(t *testing.T, g *WithT) *corev1.Namespace {

controlplane/kubeadm/internal/controllers/upgrade.go

@@ -90,7 +90,7 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane(
 
 		kubeadmCMMutators = append(kubeadmCMMutators,
 			workloadCluster.UpdateImageRepositoryInKubeadmConfigMap(imageRepository),
-			workloadCluster.UpdateFeatureGatesInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates),
+			workloadCluster.UpdateFeatureGatesInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec, parsedVersionTolerant),
 			workloadCluster.UpdateAPIServerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer),
 			workloadCluster.UpdateControllerManagerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager),
 			workloadCluster.UpdateSchedulerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Scheduler))

controlplane/kubeadm/internal/workload_cluster.go

@@ -86,6 +86,13 @@ var (
 	// NOTE: The following assumes that kubeadm version equals to Kubernetes version.
 	minVerUnversionedKubeletConfig = semver.MustParse("1.24.0")
 
+	// minKubernetesVersionControlPlaneKubeletLocalMode is the min version from which
+	// we will enable the ControlPlaneKubeletLocalMode kubeadm feature gate.
+	// Note: We have to do this with Kubernetes 1.31. Because with that version we encountered
+	// a case where it's not okay anymore to ignore the Kubernetes version skew (kubelet 1.31 uses
+	// the spec.clusterIP field selector that is only implemented in kube-apiserver >= 1.31.0).
+	minKubernetesVersionControlPlaneKubeletLocalMode = semver.MustParse("1.31.0")
+
 	// ErrControlPlaneMinNodes signals that a cluster doesn't meet the minimum required nodes
 	// to remove an etcd member.
 	ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported")
@@ -107,7 +114,7 @@ type WorkloadCluster interface {
 	ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error
 	UpdateKubernetesVersionInKubeadmConfigMap(version semver.Version) func(*bootstrapv1.ClusterConfiguration)
 	UpdateImageRepositoryInKubeadmConfigMap(imageRepository string) func(*bootstrapv1.ClusterConfiguration)
-	UpdateFeatureGatesInKubeadmConfigMap(featureGates map[string]bool) func(*bootstrapv1.ClusterConfiguration)
+	UpdateFeatureGatesInKubeadmConfigMap(kubeadmConfigSpec bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) func(*bootstrapv1.ClusterConfiguration)
 	UpdateEtcdLocalInKubeadmConfigMap(localEtcd *bootstrapv1.LocalEtcd) func(*bootstrapv1.ClusterConfiguration)
 	UpdateEtcdExternalInKubeadmConfigMap(externalEtcd *bootstrapv1.ExternalEtcd) func(*bootstrapv1.ClusterConfiguration)
 	UpdateAPIServerInKubeadmConfigMap(apiServer bootstrapv1.APIServer) func(*bootstrapv1.ClusterConfiguration)
@@ -186,11 +193,40 @@ func (w *Workload) UpdateImageRepositoryInKubeadmConfigMap(imageRepository strin
 }
 
 // UpdateFeatureGatesInKubeadmConfigMap updates the feature gates in the kubeadm config map.
-func (w *Workload) UpdateFeatureGatesInKubeadmConfigMap(featureGates map[string]bool) func(*bootstrapv1.ClusterConfiguration) {
+func (w *Workload) UpdateFeatureGatesInKubeadmConfigMap(kubeadmConfigSpec bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) func(*bootstrapv1.ClusterConfiguration) {
 	return func(c *bootstrapv1.ClusterConfiguration) {
+		// We use DeepCopy here to avoid modifying the KCP object in the apiserver.
+		kubeadmConfigSpec := kubeadmConfigSpec.DeepCopy()
+		DefaultFeatureGates(kubeadmConfigSpec, kubernetesVersion)
+
 		// Even if featureGates is nil, reset it to ClusterConfiguration
 		// to override any previously set feature gates.
-		c.FeatureGates = featureGates
+		c.FeatureGates = kubeadmConfigSpec.ClusterConfiguration.FeatureGates
+	}
+}
+
+const (
+	// ControlPlaneKubeletLocalMode is a feature gate of kubeadm that ensures
+	// kubelets only communicate with the local apiserver.
+	ControlPlaneKubeletLocalMode = "ControlPlaneKubeletLocalMode"
+)
+
+// DefaultFeatureGates defaults the feature gates field.
+func DefaultFeatureGates(kubeadmConfigSpec *bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) {
+	if kubernetesVersion.LT(minKubernetesVersionControlPlaneKubeletLocalMode) {
+		return
+	}
+
+	if kubeadmConfigSpec.ClusterConfiguration == nil {
+		kubeadmConfigSpec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{}
+	}
+
+	if kubeadmConfigSpec.ClusterConfiguration.FeatureGates == nil {
+		kubeadmConfigSpec.ClusterConfiguration.FeatureGates = map[string]bool{}
+	}
+
+	if _, ok := kubeadmConfigSpec.ClusterConfiguration.FeatureGates[ControlPlaneKubeletLocalMode]; !ok {
+		kubeadmConfigSpec.ClusterConfiguration.FeatureGates[ControlPlaneKubeletLocalMode] = true
 	}
 }