fix: register controlplane node with NoSchedule taint

Fixes #9691 This closes the race between the node registration and the moment `NodeApplyController` would apply the taint. As the taint is exactly same as added by `NodeApplyController`, it will be owned by the controller, so it can be removed if `allowSchedulingOnControlplanes` is enabled in the machine config while the cluster is running. Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
siderolabs · Nov 15, 2024 · f9697a9 · f9697a9
1 parent 30f8b5a
commit f9697a9
Show file tree

Hide file tree

Showing 8 changed files with 274 additions and 200 deletions.
diff --git a/api/resource/definitions/k8s/k8s.proto b/api/resource/definitions/k8s/k8s.proto
@@ -147,6 +147,7 @@ message KubeletConfigSpec {
   bool disable_manifests_directory = 11;
   bool enable_fs_quota_monitoring = 12;
   google.protobuf.Struct credential_provider_config = 13;
+  bool allow_scheduling_on_control_plane = 14;
 }
 
 // KubeletSpecSpec holds the source of kubelet configuration.

diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_config.go b/internal/app/machined/pkg/controllers/k8s/kubelet_config.go
@@ -78,6 +78,7 @@ func NewKubeletConfigController() *KubeletConfigController {
 				kubeletConfig.DisableManifestsDirectory = cfgProvider.Machine().Kubelet().DisableManifestsDirectory()
 				kubeletConfig.EnableFSQuotaMonitoring = cfgProvider.Machine().Features().DiskQuotaSupportEnabled()
 				kubeletConfig.CredentialProviderConfig = cfgProvider.Machine().Kubelet().CredentialProviderConfig()
+				kubeletConfig.AllowSchedulingOnControlPlane = cfgProvider.Cluster().ScheduleOnControlPlanes()
 
 				return nil
 			},

diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go b/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go
@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 	"net/netip"
+	"slices"
 	"strings"
 	"time"
 
@@ -20,6 +21,7 @@ import (
 	"github.com/siderolabs/go-kubernetes/kubernetes/compatibility"
 	"github.com/siderolabs/go-pointer"
 	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	kubeletconfig "k8s.io/kubelet/config/v1beta1"
@@ -320,6 +322,21 @@ func NewKubeletConfiguration(cfgSpec *k8s.KubeletConfigSpec, kubeletVersion comp
 	if cfgSpec.SkipNodeRegistration {
 		config.Authentication.Webhook.Enabled = pointer.To(false)
 		config.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow
+	} else if machineType.IsControlPlane() && !cfgSpec.AllowSchedulingOnControlPlane {
+		// register with taint to prevent scheduling on control plane nodes race with NodeApplyController applying the initial taint
+		// NodeApplyController will take ownership of the taint after the first successful apply
+		if slices.IndexFunc(config.RegisterWithTaints, func(t corev1.Taint) bool {
+			return t.Key == constants.LabelNodeRoleControlPlane
+		}) == -1 { // don't add the taint if it's already in the config
+			if cfgSpec.ExtraArgs["register-with-taints"] == "" { // don't clash with taints provided via extraArgs, it is deprecated on kubelet side
+				config.RegisterWithTaints = append(config.RegisterWithTaints,
+					corev1.Taint{
+						Key:    constants.LabelNodeRoleControlPlane,
+						Effect: corev1.TaintEffectNoSchedule,
+					},
+				)
+			}
+		}
 	}
 
 	// fields which can be overridden

diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go b/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go
@@ -18,6 +18,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
 	v1 "k8s.io/component-base/logs/api/v1"
@@ -423,6 +424,12 @@ func TestNewKubeletConfigurationMerge(t *testing.T) {
 			kubeletVersion: compatibility.VersionFromImageRef("ghcr.io/siderolabs/kubelet:v1.29.0"),
 			expectedOverrides: func(kc *kubeletconfig.KubeletConfiguration) {
 				kc.SystemReserved["memory"] = constants.KubeletSystemReservedMemoryControlPlane
+				kc.RegisterWithTaints = []corev1.Taint{
+					{
+						Key:    constants.LabelNodeRoleControlPlane,
+						Effect: corev1.TaintEffectNoSchedule,
+					},
+				}
 			},
 			machineType: machine.TypeControlPlane,
 		},

diff --git a/pkg/machinery/api/resource/definitions/k8s/k8s.pb.go b/pkg/machinery/api/resource/definitions/k8s/k8s.pb.go
diff --git a/pkg/machinery/api/resource/definitions/k8s/k8s_vtproto.pb.go b/pkg/machinery/api/resource/definitions/k8s/k8s_vtproto.pb.go
diff --git a/pkg/machinery/resources/k8s/kubelet_config.go b/pkg/machinery/resources/k8s/kubelet_config.go
@@ -27,19 +27,20 @@ type KubeletConfig = typed.Resource[KubeletConfigSpec, KubeletConfigExtension]
 //
 //gotagsrewrite:gen
 type KubeletConfigSpec struct {
-	Image                        string            `yaml:"image" protobuf:"1"`
-	ClusterDNS                   []string          `yaml:"clusterDNS" protobuf:"2"`
-	ClusterDomain                string            `yaml:"clusterDomain" protobuf:"3"`
-	ExtraArgs                    map[string]string `yaml:"extraArgs,omitempty" protobuf:"4"`
-	ExtraMounts                  []specs.Mount     `yaml:"extraMounts,omitempty" protobuf:"5"`
-	ExtraConfig                  map[string]any    `yaml:"extraConfig,omitempty" protobuf:"6"`
-	CloudProviderExternal        bool              `yaml:"cloudProviderExternal" protobuf:"7"`
-	DefaultRuntimeSeccompEnabled bool              `yaml:"defaultRuntimeSeccompEnabled" protobuf:"8"`
-	SkipNodeRegistration         bool              `yaml:"skipNodeRegistration" protobuf:"9"`
-	StaticPodListURL             string            `yaml:"staticPodListURL" protobuf:"10"`
-	DisableManifestsDirectory    bool              `yaml:"disableManifestsDirectory" protobuf:"11"`
-	EnableFSQuotaMonitoring      bool              `yaml:"enableFSQuotaMonitoring" protobuf:"12"`
-	CredentialProviderConfig     map[string]any    `yaml:"credentialProviderConfig,omitempty" protobuf:"13"`
+	Image                         string            `yaml:"image" protobuf:"1"`
+	ClusterDNS                    []string          `yaml:"clusterDNS" protobuf:"2"`
+	ClusterDomain                 string            `yaml:"clusterDomain" protobuf:"3"`
+	ExtraArgs                     map[string]string `yaml:"extraArgs,omitempty" protobuf:"4"`
+	ExtraMounts                   []specs.Mount     `yaml:"extraMounts,omitempty" protobuf:"5"`
+	ExtraConfig                   map[string]any    `yaml:"extraConfig,omitempty" protobuf:"6"`
+	CloudProviderExternal         bool              `yaml:"cloudProviderExternal" protobuf:"7"`
+	DefaultRuntimeSeccompEnabled  bool              `yaml:"defaultRuntimeSeccompEnabled" protobuf:"8"`
+	SkipNodeRegistration          bool              `yaml:"skipNodeRegistration" protobuf:"9"`
+	StaticPodListURL              string            `yaml:"staticPodListURL" protobuf:"10"`
+	DisableManifestsDirectory     bool              `yaml:"disableManifestsDirectory" protobuf:"11"`
+	EnableFSQuotaMonitoring       bool              `yaml:"enableFSQuotaMonitoring" protobuf:"12"`
+	CredentialProviderConfig      map[string]any    `yaml:"credentialProviderConfig,omitempty" protobuf:"13"`
+	AllowSchedulingOnControlPlane bool              `yaml:"allowSchedulingOnControlPlane" protobuf:"14"`
 }
 
 // NewKubeletConfig initializes an empty KubeletConfig resource.

diff --git a/website/content/v1.9/reference/api.md b/website/content/v1.9/reference/api.md
@@ -2693,6 +2693,7 @@ KubeletConfigSpec holds the source of kubelet configuration.
 | disable_manifests_directory | [bool](#bool) |  |  |
 | enable_fs_quota_monitoring | [bool](#bool) |  |  |
 | credential_provider_config | [google.protobuf.Struct](#google.protobuf.Struct) |  |  |
+| allow_scheduling_on_control_plane | [bool](#bool) |  |  |