Merge pull request #5040 from zhzhuang-zju/grpctlsauth

Adding TLS Certificate Authentication to gRPC

artifacts/deploy/karmada-descheduler.yaml

@@ -27,6 +27,9 @@ spec:
             - /bin/karmada-descheduler
             - --kubeconfig=/etc/kubeconfig
             - --bind-address=0.0.0.0
+            - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
+            - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
+            - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
             - --v=4
           livenessProbe:
             httpGet:
@@ -38,10 +41,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: kubeconfig
               subPath: kubeconfig
               mountPath: /etc/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: kubeconfig
           secret:
             secretName: kubeconfig

artifacts/deploy/karmada-scheduler-estimator.yaml

@@ -27,6 +27,9 @@ spec:
             - /bin/karmada-scheduler-estimator
             - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig
             - --cluster-name={{member_cluster_name}}
+            - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt
+            - --grpc-auth-key-file=/etc/karmada/pki/karmada.key
+            - --grpc-client-ca-file=/etc/karmada/pki/ca.crt
           livenessProbe:
             httpGet:
               path: /healthz
@@ -37,10 +40,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: member-kubeconfig
               subPath: {{member_cluster_name}}-kubeconfig
               mountPath: /etc/{{member_cluster_name}}-kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: member-kubeconfig
           secret:
             secretName: {{member_cluster_name}}-kubeconfig

artifacts/deploy/karmada-scheduler.yaml

@@ -38,12 +38,21 @@ spec:
             - --bind-address=0.0.0.0
             - --secure-port=10351
             - --enable-scheduler-estimator=true
+            - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
+            - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
+            - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
             - --v=4
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: kubeconfig
               subPath: kubeconfig
               mountPath: /etc/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: kubeconfig
           secret:
             secretName: kubeconfig

cmd/descheduler/app/options/options.go

@@ -62,6 +62,14 @@ type Options struct {
 	SchedulerEstimatorServicePrefix string
 	// SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at.
 	SchedulerEstimatorPort int
+	// SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication.
+	SchedulerEstimatorCertFile string
+	// SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication.
+	SchedulerEstimatorKeyFile string
+	// SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication.
+	SchedulerEstimatorCaFile string
+	// InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name.
+	InsecureSkipEstimatorVerify bool
 	// DeschedulingInterval specifies time interval for descheduler to run.
 	DeschedulingInterval metav1.Duration
 	// UnschedulableThreshold specifies the period of pod unschedulable condition.
@@ -99,6 +107,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.")
 	fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
 	fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
+	fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.")
+	fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.")
+	fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.")
+	fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.")
 	fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
 	fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.")
 	fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.")

cmd/scheduler-estimator/app/options/options.go

@@ -40,6 +40,14 @@ type Options struct {
 	SecurePort int
 	// ServerPort is the port that the server gRPC serves at.
 	ServerPort int
+	// InsecureSkipGrpcClientVerify controls whether verifies the grpc client's certificate chain and host name.
+	InsecureSkipGrpcClientVerify bool
+	// GrpcAuthCertFile SSL certification file used for grpc SSL/TLS connections.
+	GrpcAuthCertFile string
+	// GrpcAuthKeyFile SSL key file used for grpc SSL/TLS connections.
+	GrpcAuthKeyFile string
+	// GrpcClientCaFile SSL Certificate Authority file used to verify grpc client certificates on incoming requests.
+	GrpcClientCaFile string
 	// ClusterAPIQPS is the QPS to use while talking with cluster kube-apiserver.
 	ClusterAPIQPS float32
 	// ClusterAPIBurst is the burst to allow while talking with cluster kube-apiserver.
@@ -64,6 +72,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.ClusterName, "cluster-name", o.ClusterName, "Name of member cluster that the estimator serves for.")
 	fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.")
 	fs.IntVar(&o.ServerPort, "server-port", defaultServerPort, "The secure port on which to serve gRPC.")
+	fs.StringVar(&o.GrpcAuthCertFile, "grpc-auth-cert-file", "", "SSL certification file used for grpc SSL/TLS connections.")
+	fs.StringVar(&o.GrpcAuthKeyFile, "grpc-auth-key-file", "", "SSL key file used for grpc SSL/TLS connections.")
+	fs.BoolVar(&o.InsecureSkipGrpcClientVerify, "insecure-skip-grpc-client-verify", false, "If set to true, the estimator will not verify the grpc client's certificate chain and host name. When the relevant certificates are not configured, it will not take effect.")
+	fs.StringVar(&o.GrpcClientCaFile, "grpc-client-ca-file", "", "SSL Certificate Authority file used to verify grpc client certificates on incoming requests if --client-cert-auth flag is set.")
 	fs.IntVar(&o.SecurePort, "secure-port", defaultHealthzPort, "The secure port on which to serve HTTPS.")
 	fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver.")
 	fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver.")

cmd/scheduler/app/options/options.go

@@ -71,6 +71,14 @@ type Options struct {
 	SchedulerEstimatorServicePrefix string
 	// SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at.
 	SchedulerEstimatorPort int
+	// InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name.
+	InsecureSkipEstimatorVerify bool
+	// SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication.
+	SchedulerEstimatorCertFile string
+	// SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication.
+	SchedulerEstimatorKeyFile string
+	// SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication.
+	SchedulerEstimatorCaFile string
 
 	// EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters.
 	EnableEmptyWorkloadPropagation bool
@@ -138,6 +146,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
 	fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
 	fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
+	fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.")
+	fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.")
+	fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.")
+	fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.")
 	fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.")
 	fs.StringSliceVar(&o.Plugins, "plugins", []string{"*"},
 		fmt.Sprintf("A list of plugins to enable. '*' enables all build-in and customized plugins, 'foo' enables the plugin named 'foo', '*,-foo' disables the plugin named 'foo'.\nAll build-in plugins: %s.", strings.Join(frameworkplugins.NewInTreeRegistry().FactoryNames(), ",")))

cmd/scheduler/app/scheduler.go

@@ -170,7 +170,7 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt
 		scheduler.WithEnableSchedulerEstimator(opts.EnableSchedulerEstimator),
 		scheduler.WithDisableSchedulerEstimatorInPullMode(opts.DisableSchedulerEstimatorInPullMode),
 		scheduler.WithSchedulerEstimatorServicePrefix(opts.SchedulerEstimatorServicePrefix),
-		scheduler.WithSchedulerEstimatorPort(opts.SchedulerEstimatorPort),
+		scheduler.WithSchedulerEstimatorConnection(opts.SchedulerEstimatorPort, opts.SchedulerEstimatorCertFile, opts.SchedulerEstimatorKeyFile, opts.SchedulerEstimatorCaFile, opts.InsecureSkipEstimatorVerify),
 		scheduler.WithSchedulerEstimatorTimeout(opts.SchedulerEstimatorTimeout),
 		scheduler.WithEnableEmptyWorkloadPropagation(opts.EnableEmptyWorkloadPropagation),
 		scheduler.WithEnableSchedulerPlugin(opts.Plugins),

operator/pkg/controlplane/manifests.go

@@ -188,6 +188,9 @@ spec:
         - --secure-port=10351
         - --enable-scheduler-estimator=true
         - --leader-elect-resource-namespace={{ .SystemNamespace }}
+        - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
+        - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
+        - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
         - --v=4
         livenessProbe:
           httpGet:
@@ -199,10 +202,16 @@ spec:
           periodSeconds: 15
           timeoutSeconds: 5
         volumeMounts:
+        - name: karmada-certs
+          mountPath: /etc/karmada/pki
+          readOnly: true
         - name: kubeconfig
           subPath: kubeconfig
           mountPath: /etc/karmada/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: {{ .KarmadaCertsSecret }}
         - name: kubeconfig
           secret:
             secretName: {{ .KubeconfigSecret }}
@@ -241,6 +250,9 @@ spec:
         - --kubeconfig=/etc/karmada/kubeconfig
         - --bind-address=0.0.0.0
         - --leader-elect-resource-namespace={{ .SystemNamespace }}
+        - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
+        - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
+        - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
         - --v=4
         livenessProbe:
           httpGet:
@@ -252,10 +264,16 @@ spec:
           periodSeconds: 15
           timeoutSeconds: 5
         volumeMounts:
+        - name: karmada-certs
+          mountPath: /etc/karmada/pki
+          readOnly: true
         - name: kubeconfig
           subPath: kubeconfig
           mountPath: /etc/karmada/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: {{ .KarmadaCertsSecret }}
         - name: kubeconfig
           secret:
             secretName: {{ .KubeconfigSecret }}

pkg/descheduler/descheduler.go

@@ -45,6 +45,7 @@ import (
 	"github.com/karmada-io/karmada/pkg/util"
 	"github.com/karmada-io/karmada/pkg/util/fedinformer"
 	"github.com/karmada-io/karmada/pkg/util/gclient"
+	"github.com/karmada-io/karmada/pkg/util/grpcconnection"
 )
 
 const (
@@ -65,7 +66,7 @@ type Descheduler struct {
 
 	schedulerEstimatorCache         *estimatorclient.SchedulerEstimatorCache
 	schedulerEstimatorServicePrefix string
-	schedulerEstimatorPort          int
+	schedulerEstimatorClientConfig  *grpcconnection.ClientConfig
 	schedulerEstimatorWorker        util.AsyncWorker
 
 	unschedulableThreshold time.Duration
@@ -77,15 +78,21 @@ type Descheduler struct {
 func NewDescheduler(karmadaClient karmadaclientset.Interface, kubeClient kubernetes.Interface, opts *options.Options) *Descheduler {
 	factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0)
 	desched := &Descheduler{
-		KarmadaClient:                   karmadaClient,
-		KubeClient:                      kubeClient,
-		informerFactory:                 factory,
-		bindingInformer:                 factory.Work().V1alpha2().ResourceBindings().Informer(),
-		bindingLister:                   factory.Work().V1alpha2().ResourceBindings().Lister(),
-		clusterInformer:                 factory.Cluster().V1alpha1().Clusters().Informer(),
-		clusterLister:                   factory.Cluster().V1alpha1().Clusters().Lister(),
-		schedulerEstimatorCache:         estimatorclient.NewSchedulerEstimatorCache(),
-		schedulerEstimatorPort:          opts.SchedulerEstimatorPort,
+		KarmadaClient:           karmadaClient,
+		KubeClient:              kubeClient,
+		informerFactory:         factory,
+		bindingInformer:         factory.Work().V1alpha2().ResourceBindings().Informer(),
+		bindingLister:           factory.Work().V1alpha2().ResourceBindings().Lister(),
+		clusterInformer:         factory.Cluster().V1alpha1().Clusters().Informer(),
+		clusterLister:           factory.Cluster().V1alpha1().Clusters().Lister(),
+		schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(),
+		schedulerEstimatorClientConfig: &grpcconnection.ClientConfig{
+			InsecureSkipServerVerify: opts.InsecureSkipEstimatorVerify,
+			ServerAuthCAFile:         opts.SchedulerEstimatorCaFile,
+			CertFile:                 opts.SchedulerEstimatorCertFile,
+			KeyFile:                  opts.SchedulerEstimatorKeyFile,
+			TargetPort:               opts.SchedulerEstimatorPort,
+		},
 		schedulerEstimatorServicePrefix: opts.SchedulerEstimatorServicePrefix,
 		unschedulableThreshold:          opts.UnschedulableThreshold.Duration,
 		deschedulingInterval:            opts.DeschedulingInterval.Duration,
@@ -273,7 +280,7 @@ func (d *Descheduler) establishEstimatorConnections() {
 		return
 	}
 	for i := range clusterList.Items {
-		if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort); err != nil {
+		if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig); err != nil {
 			klog.Error(err)
 		}
 	}
@@ -293,7 +300,7 @@ func (d *Descheduler) reconcileEstimatorConnection(key util.QueueKey) error {
 		}
 		return err
 	}
-	return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort)
+	return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig)
 }
 
 func (d *Descheduler) recordDescheduleResultEventForResourceBinding(rb *workv1alpha2.ResourceBinding, message string, err error) {

pkg/estimator/client/cache.go

@@ -27,6 +27,7 @@ import (
 
 	estimatorservice "github.com/karmada-io/karmada/pkg/estimator/service"
 	"github.com/karmada-io/karmada/pkg/util"
+	"github.com/karmada-io/karmada/pkg/util/grpcconnection"
 	"github.com/karmada-io/karmada/pkg/util/names"
 )
 
@@ -96,19 +97,19 @@ func (c *SchedulerEstimatorCache) GetClient(name string) (estimatorservice.Estim
 }
 
 // EstablishConnection establishes a new gRPC connection with the specified cluster scheduler estimator.
-func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, port int) error {
+func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, grpcConfig *grpcconnection.ClientConfig) error {
 	if estimatorCache.IsEstimatorExist(name) {
 		return nil
 	}
 
 	serverAddr, err := resolveCluster(kubeClient, util.NamespaceKarmadaSystem,
-		names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(port))
+		names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(grpcConfig.TargetPort))
 	if err != nil {
 		return err
 	}
 
 	klog.Infof("Start dialing estimator server(%s) of cluster(%s).", serverAddr, name)
-	cc, err := util.Dial(serverAddr, 5*time.Second)
+	cc, err := grpcConfig.DialWithTimeOut(serverAddr, 5*time.Second)
 	if err != nil {
 		klog.Errorf("Failed to dial cluster(%s): %v.", name, err)
 		return err

pkg/estimator/client/service.go

@@ -41,7 +41,7 @@ func resolveCluster(kubeClient kubernetes.Interface, namespace, id string, port
 			 * But the Service resource is defined in Host Kubernetes Cluster. So we cannot get its content here.
 			 * The best thing we can do is just glue host:port together, and try to connect to it.
 			 */
-			return net.JoinHostPort(id, fmt.Sprintf("%d", port)), nil
+			return net.JoinHostPort(fmt.Sprintf("%s.%s.svc.cluster.local", id, namespace), fmt.Sprintf("%d", port)), nil
 		}
 
 		return "", err