diff --git a/controlplane/kubeadm/internal/controllers/controller_test.go b/controlplane/kubeadm/internal/controllers/controller_test.go index 5a3c8971f632..da71a74ff1b3 100644 --- a/controlplane/kubeadm/internal/controllers/controller_test.go +++ b/controlplane/kubeadm/internal/controllers/controller_test.go @@ -2351,7 +2351,7 @@ func createClusterWithControlPlane(namespace string) (*clusterv1.Cluster, *contr }, }, Replicas: ptr.To[int32](int32(3)), - Version: "v1.16.6", + Version: "v1.31.0", RolloutStrategy: &controlplanev1.RolloutStrategy{ Type: "RollingUpdate", RollingUpdate: &controlplanev1.RollingUpdate{ diff --git a/controlplane/kubeadm/internal/controllers/scale.go b/controlplane/kubeadm/internal/controllers/scale.go index 3d43cdb5d9c8..43bcb37d87cc 100644 --- a/controlplane/kubeadm/internal/controllers/scale.go +++ b/controlplane/kubeadm/internal/controllers/scale.go @@ -33,12 +33,22 @@ import ( "sigs.k8s.io/cluster-api/controlplane/kubeadm/internal" "sigs.k8s.io/cluster-api/util/collections" "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/version" ) func (r *KubeadmControlPlaneReconciler) initializeControlPlane(ctx context.Context, controlPlane *internal.ControlPlane) (ctrl.Result, error) { logger := ctrl.LoggerFrom(ctx) bootstrapSpec := controlPlane.InitialControlPlaneConfig() + + // We intentionally only parse major/minor/patch so that the subsequent code + // also already applies to beta versions of new releases. + parsedVersionTolerant, err := version.ParseMajorMinorPatchTolerant(controlPlane.KCP.Spec.Version) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version) + } + internal.DefaultFeatureGates(bootstrapSpec, parsedVersionTolerant) + fd, err := controlPlane.NextFailureDomainForScaleUp(ctx) if err != nil { return ctrl.Result{}, err @@ -64,6 +74,15 @@ func (r *KubeadmControlPlaneReconciler) scaleUpControlPlane(ctx context.Context, // Create the bootstrap configuration bootstrapSpec := controlPlane.JoinControlPlaneConfig() + + // We intentionally only parse major/minor/patch so that the subsequent code + // also already applies to beta versions of new releases. + parsedVersionTolerant, err := version.ParseMajorMinorPatchTolerant(controlPlane.KCP.Spec.Version) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version) + } + internal.DefaultFeatureGates(bootstrapSpec, parsedVersionTolerant) + fd, err := controlPlane.NextFailureDomainForScaleUp(ctx) if err != nil { return ctrl.Result{}, err diff --git a/controlplane/kubeadm/internal/controllers/scale_test.go b/controlplane/kubeadm/internal/controllers/scale_test.go index 3e9ae0095a5c..00f0cbb686fd 100644 --- a/controlplane/kubeadm/internal/controllers/scale_test.go +++ b/controlplane/kubeadm/internal/controllers/scale_test.go @@ -102,6 +102,11 @@ func TestKubeadmControlPlaneReconciler_initializeControlPlane(t *testing.T) { g.Expect(machineList.Items[0].Spec.Bootstrap.ConfigRef.Name).To(Equal(machineList.Items[0].Name)) g.Expect(machineList.Items[0].Spec.Bootstrap.ConfigRef.APIVersion).To(Equal(bootstrapv1.GroupVersion.String())) g.Expect(machineList.Items[0].Spec.Bootstrap.ConfigRef.Kind).To(Equal("KubeadmConfig")) + + kubeadmConfig := &bootstrapv1.KubeadmConfig{} + bootstrapRef := machineList.Items[0].Spec.Bootstrap.ConfigRef + g.Expect(env.GetAPIReader().Get(ctx, client.ObjectKey{Namespace: bootstrapRef.Namespace, Name: bootstrapRef.Name}, kubeadmConfig)).To(Succeed()) + g.Expect(kubeadmConfig.Spec.ClusterConfiguration.FeatureGates).To(BeComparableTo(map[string]bool{internal.ControlPlaneKubeletLocalMode: true})) } func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) { @@ -165,6 +170,11 @@ func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) { // Note: expected length is 1 because only the newly created machine is on API server. Other machines are // in-memory only during the test. g.Expect(controlPlaneMachines.Items).To(HaveLen(1)) + + kubeadmConfig := &bootstrapv1.KubeadmConfig{} + bootstrapRef := controlPlaneMachines.Items[0].Spec.Bootstrap.ConfigRef + g.Expect(env.GetAPIReader().Get(ctx, client.ObjectKey{Namespace: bootstrapRef.Namespace, Name: bootstrapRef.Name}, kubeadmConfig)).To(Succeed()) + g.Expect(kubeadmConfig.Spec.ClusterConfiguration.FeatureGates).To(BeComparableTo(map[string]bool{internal.ControlPlaneKubeletLocalMode: true})) }) t.Run("does not create a control plane Machine if preflight checks fail", func(t *testing.T) { setup := func(t *testing.T, g *WithT) *corev1.Namespace { diff --git a/controlplane/kubeadm/internal/controllers/upgrade.go b/controlplane/kubeadm/internal/controllers/upgrade.go index ae9823d6154d..20a1589294a6 100644 --- a/controlplane/kubeadm/internal/controllers/upgrade.go +++ b/controlplane/kubeadm/internal/controllers/upgrade.go @@ -90,7 +90,7 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane( kubeadmCMMutators = append(kubeadmCMMutators, workloadCluster.UpdateImageRepositoryInKubeadmConfigMap(imageRepository), - workloadCluster.UpdateFeatureGatesInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates), + workloadCluster.UpdateFeatureGatesInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec, parsedVersionTolerant), workloadCluster.UpdateAPIServerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer), workloadCluster.UpdateControllerManagerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager), workloadCluster.UpdateSchedulerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Scheduler)) diff --git a/controlplane/kubeadm/internal/workload_cluster.go b/controlplane/kubeadm/internal/workload_cluster.go index 9034dd1e05e8..0d35ae2d5306 100644 --- a/controlplane/kubeadm/internal/workload_cluster.go +++ b/controlplane/kubeadm/internal/workload_cluster.go @@ -86,6 +86,13 @@ var ( // NOTE: The following assumes that kubeadm version equals to Kubernetes version. minVerUnversionedKubeletConfig = semver.MustParse("1.24.0") + // minKubernetesVersionControlPlaneKubeletLocalMode is the min version from which + // we will enable the ControlPlaneKubeletLocalMode kubeadm feature gate. + // Note: We have to do this with Kubernetes 1.31. Because with that version we encountered + // a case where it's not okay anymore to ignore the Kubernetes version skew (kubelet 1.31 uses + // the spec.clusterIP field selector that is only implemented in kube-apiserver >= 1.31.0). + minKubernetesVersionControlPlaneKubeletLocalMode = semver.MustParse("1.31.0") + // ErrControlPlaneMinNodes signals that a cluster doesn't meet the minimum required nodes // to remove an etcd member. ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported") @@ -107,7 +114,7 @@ type WorkloadCluster interface { ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error UpdateKubernetesVersionInKubeadmConfigMap(version semver.Version) func(*bootstrapv1.ClusterConfiguration) UpdateImageRepositoryInKubeadmConfigMap(imageRepository string) func(*bootstrapv1.ClusterConfiguration) - UpdateFeatureGatesInKubeadmConfigMap(featureGates map[string]bool) func(*bootstrapv1.ClusterConfiguration) + UpdateFeatureGatesInKubeadmConfigMap(kubeadmConfigSpec bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) func(*bootstrapv1.ClusterConfiguration) UpdateEtcdLocalInKubeadmConfigMap(localEtcd *bootstrapv1.LocalEtcd) func(*bootstrapv1.ClusterConfiguration) UpdateEtcdExternalInKubeadmConfigMap(externalEtcd *bootstrapv1.ExternalEtcd) func(*bootstrapv1.ClusterConfiguration) UpdateAPIServerInKubeadmConfigMap(apiServer bootstrapv1.APIServer) func(*bootstrapv1.ClusterConfiguration) @@ -186,11 +193,40 @@ func (w *Workload) UpdateImageRepositoryInKubeadmConfigMap(imageRepository strin } // UpdateFeatureGatesInKubeadmConfigMap updates the feature gates in the kubeadm config map. -func (w *Workload) UpdateFeatureGatesInKubeadmConfigMap(featureGates map[string]bool) func(*bootstrapv1.ClusterConfiguration) { +func (w *Workload) UpdateFeatureGatesInKubeadmConfigMap(kubeadmConfigSpec bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) func(*bootstrapv1.ClusterConfiguration) { return func(c *bootstrapv1.ClusterConfiguration) { + // We use DeepCopy here to avoid modifying the KCP object in the apiserver. + kubeadmConfigSpec := kubeadmConfigSpec.DeepCopy() + DefaultFeatureGates(kubeadmConfigSpec, kubernetesVersion) + // Even if featureGates is nil, reset it to ClusterConfiguration // to override any previously set feature gates. - c.FeatureGates = featureGates + c.FeatureGates = kubeadmConfigSpec.ClusterConfiguration.FeatureGates + } +} + +const ( + // ControlPlaneKubeletLocalMode is a feature gate of kubeadm that ensures + // kubelets only communicate with the local apiserver. + ControlPlaneKubeletLocalMode = "ControlPlaneKubeletLocalMode" +) + +// DefaultFeatureGates defaults the feature gates field. +func DefaultFeatureGates(kubeadmConfigSpec *bootstrapv1.KubeadmConfigSpec, kubernetesVersion semver.Version) { + if kubernetesVersion.LT(minKubernetesVersionControlPlaneKubeletLocalMode) { + return + } + + if kubeadmConfigSpec.ClusterConfiguration == nil { + kubeadmConfigSpec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{} + } + + if kubeadmConfigSpec.ClusterConfiguration.FeatureGates == nil { + kubeadmConfigSpec.ClusterConfiguration.FeatureGates = map[string]bool{} + } + + if _, ok := kubeadmConfigSpec.ClusterConfiguration.FeatureGates[ControlPlaneKubeletLocalMode]; !ok { + kubeadmConfigSpec.ClusterConfiguration.FeatureGates[ControlPlaneKubeletLocalMode] = true } } diff --git a/controlplane/kubeadm/internal/workload_cluster_test.go b/controlplane/kubeadm/internal/workload_cluster_test.go index eb475a89ca21..9861419d99b4 100644 --- a/controlplane/kubeadm/internal/workload_cluster_test.go +++ b/controlplane/kubeadm/internal/workload_cluster_test.go @@ -1266,16 +1266,30 @@ func TestUpdateFeatureGatesInKubeadmConfigMap(t *testing.T) { tests := []struct { name string clusterConfigurationData string - newFeatureGates map[string]bool - wantFeatureGates map[string]bool + kubernetesVersion semver.Version + newClusterConfiguration *bootstrapv1.ClusterConfiguration + wantClusterConfiguration *bootstrapv1.ClusterConfiguration }{ { name: "it updates feature gates", clusterConfigurationData: utilyaml.Raw(` apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration`), - newFeatureGates: map[string]bool{"EtcdLearnerMode": true}, - wantFeatureGates: map[string]bool{"EtcdLearnerMode": true}, + kubernetesVersion: semver.MustParse("1.19.1"), + newClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + }, + }, + wantClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kubeadm.k8s.io/v1beta2", + Kind: "ClusterConfiguration", + }, + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + }, + }, }, { name: "it should override feature gates even if new value is nil", @@ -1285,8 +1299,98 @@ func TestUpdateFeatureGatesInKubeadmConfigMap(t *testing.T) { featureGates: EtcdLearnerMode: true `), - newFeatureGates: nil, - wantFeatureGates: nil, + kubernetesVersion: semver.MustParse("1.19.1"), + newClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: nil, + }, + wantClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kubeadm.k8s.io/v1beta2", + Kind: "ClusterConfiguration", + }, + FeatureGates: nil, + }, + }, + { + name: "it should not add ControlPlaneKubeletLocalMode feature gate for 1.30", + clusterConfigurationData: utilyaml.Raw(` + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration`), + kubernetesVersion: semver.MustParse("1.30.0"), + newClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: nil, + }, + wantClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kubeadm.k8s.io/v1beta4", + Kind: "ClusterConfiguration", + }, + FeatureGates: nil, + }, + }, + { + name: "it should add ControlPlaneKubeletLocalMode feature gate for 1.31.0", + clusterConfigurationData: utilyaml.Raw(` + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration`), + kubernetesVersion: semver.MustParse("1.31.0"), + newClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: nil, + }, + wantClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kubeadm.k8s.io/v1beta4", + Kind: "ClusterConfiguration", + }, + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: true, + }, + }, + }, + { + name: "it should add ControlPlaneKubeletLocalMode feature gate for 1.31.0 if other feature gate is set", + clusterConfigurationData: utilyaml.Raw(` + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration`), + kubernetesVersion: semver.MustParse("1.31.0"), + newClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + }, + }, + wantClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kubeadm.k8s.io/v1beta4", + Kind: "ClusterConfiguration", + }, + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: true, + "EtcdLearnerMode": true, + }, + }, + }, + { + name: "it should preserve ControlPlaneKubeletLocalMode false feature gate for 1.31.0", + clusterConfigurationData: utilyaml.Raw(` + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration`), + kubernetesVersion: semver.MustParse("1.31.0"), + newClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + ControlPlaneKubeletLocalMode: false, + }, + }, + wantClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kubeadm.k8s.io/v1beta4", + Kind: "ClusterConfiguration", + }, + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: false, + "EtcdLearnerMode": true, + }, + }, }, } @@ -1306,7 +1410,7 @@ func TestUpdateFeatureGatesInKubeadmConfigMap(t *testing.T) { w := &Workload{ Client: fakeClient, } - err := w.UpdateClusterConfiguration(ctx, semver.MustParse("1.19.1"), w.UpdateFeatureGatesInKubeadmConfigMap(tt.newFeatureGates)) + err := w.UpdateClusterConfiguration(ctx, tt.kubernetesVersion, w.UpdateFeatureGatesInKubeadmConfigMap(bootstrapv1.KubeadmConfigSpec{ClusterConfiguration: tt.newClusterConfiguration}, tt.kubernetesVersion)) g.Expect(err).ToNot(HaveOccurred()) var actualConfig corev1.ConfigMap @@ -1316,12 +1420,132 @@ func TestUpdateFeatureGatesInKubeadmConfigMap(t *testing.T) { &actualConfig, )).To(Succeed()) - actualConfiguration := bootstrapv1.ClusterConfiguration{} - err = yaml.Unmarshal([]byte(actualConfig.Data[clusterConfigurationKey]), &actualConfiguration) + actualConfiguration := &bootstrapv1.ClusterConfiguration{} + err = yaml.Unmarshal([]byte(actualConfig.Data[clusterConfigurationKey]), actualConfiguration) if err != nil { return } - g.Expect(actualConfiguration.FeatureGates).Should(Equal(tt.wantFeatureGates)) + g.Expect(actualConfiguration).Should(BeComparableTo(tt.wantClusterConfiguration)) + }) + } +} + +func TestDefaultFeatureGates(t *testing.T) { + tests := []struct { + name string + kubernetesVersion semver.Version + kubeadmConfigSpec *bootstrapv1.KubeadmConfigSpec + wantKubeadmConfigSpec *bootstrapv1.KubeadmConfigSpec + }{ + { + name: "don't default ControlPlaneKubeletLocalMode for 1.30", + kubernetesVersion: semver.MustParse("1.30.99"), + kubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + }, + }, + }, + wantKubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + }, + }, + }, + }, + { + name: "default ControlPlaneKubeletLocalMode for 1.31", + kubernetesVersion: semver.MustParse("1.31.0"), + kubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: nil, + }, + wantKubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: true, + }, + }, + }, + }, + { + name: "default ControlPlaneKubeletLocalMode for 1.31", + kubernetesVersion: semver.MustParse("1.31.0"), + kubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: nil, + }, + }, + wantKubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: true, + }, + }, + }, + }, + { + name: "default ControlPlaneKubeletLocalMode for 1.31", + kubernetesVersion: semver.MustParse("1.31.0"), + kubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{}, + }, + }, + wantKubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: true, + }, + }, + }, + }, + { + name: "default ControlPlaneKubeletLocalMode for 1.31", + kubernetesVersion: semver.MustParse("1.31.0"), + kubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + "EtcdLearnerMode": true, + }, + }, + }, + wantKubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: true, + "EtcdLearnerMode": true, + }, + }, + }, + }, + { + name: "don't default ControlPlaneKubeletLocalMode for 1.31 if already set to false", + kubernetesVersion: semver.MustParse("1.31.0"), + kubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: false, + }, + }, + }, + wantKubeadmConfigSpec: &bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ + FeatureGates: map[string]bool{ + ControlPlaneKubeletLocalMode: false, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + DefaultFeatureGates(tt.kubeadmConfigSpec, tt.kubernetesVersion) + g.Expect(tt.wantKubeadmConfigSpec).Should(BeComparableTo(tt.kubeadmConfigSpec)) }) } } diff --git a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml index 754c5503d417..ced2d8a77f5b 100644 --- a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml +++ b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml @@ -480,20 +480,6 @@ spec: path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/v" valueFrom: variable: kubeletLogLevel - - name: kubeadmFeatureGate - description: "Sets the ControlPlaneKubeletLocalMode feature gate for Kubernetes >= 1.31" - definitions: - - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/featureGates" - value: - ControlPlaneKubeletLocalMode: true - enabledIf: '{{ semverCompare ">= v1.31.0-0" .builtin.controlPlane.version }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerClusterTemplate