diff --git a/artifacts/deploy/karmada-descheduler.yaml b/artifacts/deploy/karmada-descheduler.yaml index c451a37b4312..53d0d189bd7c 100644 --- a/artifacts/deploy/karmada-descheduler.yaml +++ b/artifacts/deploy/karmada-descheduler.yaml @@ -27,6 +27,9 @@ spec: - /bin/karmada-descheduler - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -38,10 +41,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: kubeconfig secret: secretName: kubeconfig diff --git a/artifacts/deploy/karmada-scheduler-estimator.yaml b/artifacts/deploy/karmada-scheduler-estimator.yaml index ce4af9361e1a..85d6111f5aae 100644 --- a/artifacts/deploy/karmada-scheduler-estimator.yaml +++ b/artifacts/deploy/karmada-scheduler-estimator.yaml @@ -27,6 +27,9 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig - --cluster-name={{member_cluster_name}} + - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt + - --grpc-auth-key-file=/etc/karmada/pki/karmada.key + - --grpc-client-ca-file=/etc/karmada/pki/ca.crt livenessProbe: httpGet: path: /healthz @@ -37,10 +40,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: member-kubeconfig subPath: {{member_cluster_name}}-kubeconfig mountPath: /etc/{{member_cluster_name}}-kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: member-kubeconfig secret: secretName: {{member_cluster_name}}-kubeconfig diff --git a/artifacts/deploy/karmada-scheduler.yaml b/artifacts/deploy/karmada-scheduler.yaml index 1401e7a4cca0..f863fba55a3a 100644 --- a/artifacts/deploy/karmada-scheduler.yaml +++ b/artifacts/deploy/karmada-scheduler.yaml @@ -38,12 +38,21 @@ spec: - --bind-address=0.0.0.0 - --secure-port=10351 - --enable-scheduler-estimator=true + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: kubeconfig secret: secretName: kubeconfig diff --git a/cmd/descheduler/app/options/options.go b/cmd/descheduler/app/options/options.go index 0dd0d0b816cb..67e8a54d4e34 100644 --- a/cmd/descheduler/app/options/options.go +++ b/cmd/descheduler/app/options/options.go @@ -62,6 +62,14 @@ type Options struct { SchedulerEstimatorServicePrefix string // SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. SchedulerEstimatorPort int + // SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication. + SchedulerEstimatorCertFile string + // SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication. + SchedulerEstimatorKeyFile string + // SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication. + SchedulerEstimatorCaFile string + // InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name. + InsecureSkipEstimatorVerify bool // DeschedulingInterval specifies time interval for descheduler to run. DeschedulingInterval metav1.Duration // UnschedulableThreshold specifies the period of pod unschedulable condition. @@ -99,6 +107,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.") fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.") fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.") + fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.") + fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.") + fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.") + fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.") fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name") fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.") fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.") diff --git a/cmd/scheduler-estimator/app/options/options.go b/cmd/scheduler-estimator/app/options/options.go index f10bf3ca9cb2..1d6ee5d26435 100644 --- a/cmd/scheduler-estimator/app/options/options.go +++ b/cmd/scheduler-estimator/app/options/options.go @@ -40,6 +40,14 @@ type Options struct { SecurePort int // ServerPort is the port that the server gRPC serves at. ServerPort int + // InsecureSkipGrpcClientVerify controls whether verifies the grpc client's certificate chain and host name. + InsecureSkipGrpcClientVerify bool + // GrpcAuthCertFile SSL certification file used for grpc SSL/TLS connections. + GrpcAuthCertFile string + // GrpcAuthKeyFile SSL key file used for grpc SSL/TLS connections. + GrpcAuthKeyFile string + // GrpcClientCaFile SSL Certificate Authority file used to verify grpc client certificates on incoming requests. + GrpcClientCaFile string // ClusterAPIQPS is the QPS to use while talking with cluster kube-apiserver. ClusterAPIQPS float32 // ClusterAPIBurst is the burst to allow while talking with cluster kube-apiserver. @@ -64,6 +72,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.ClusterName, "cluster-name", o.ClusterName, "Name of member cluster that the estimator serves for.") fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.") fs.IntVar(&o.ServerPort, "server-port", defaultServerPort, "The secure port on which to serve gRPC.") + fs.StringVar(&o.GrpcAuthCertFile, "grpc-auth-cert-file", "", "SSL certification file used for grpc SSL/TLS connections.") + fs.StringVar(&o.GrpcAuthKeyFile, "grpc-auth-key-file", "", "SSL key file used for grpc SSL/TLS connections.") + fs.BoolVar(&o.InsecureSkipGrpcClientVerify, "insecure-skip-grpc-client-verify", false, "If set to true, the estimator will not verify the grpc client's certificate chain and host name. When the relevant certificates are not configured, it will not take effect.") + fs.StringVar(&o.GrpcClientCaFile, "grpc-client-ca-file", "", "SSL Certificate Authority file used to verify grpc client certificates on incoming requests if --client-cert-auth flag is set.") fs.IntVar(&o.SecurePort, "secure-port", defaultHealthzPort, "The secure port on which to serve HTTPS.") fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver.") fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver.") diff --git a/cmd/scheduler/app/options/options.go b/cmd/scheduler/app/options/options.go index bcb6b2e59634..ac994775253c 100644 --- a/cmd/scheduler/app/options/options.go +++ b/cmd/scheduler/app/options/options.go @@ -71,6 +71,14 @@ type Options struct { SchedulerEstimatorServicePrefix string // SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. SchedulerEstimatorPort int + // InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name. + InsecureSkipEstimatorVerify bool + // SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication. + SchedulerEstimatorCertFile string + // SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication. + SchedulerEstimatorKeyFile string + // SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication. + SchedulerEstimatorCaFile string // EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters. EnableEmptyWorkloadPropagation bool @@ -138,6 +146,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.") fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name") fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.") + fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.") + fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.") + fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.") + fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.") fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.") fs.StringSliceVar(&o.Plugins, "plugins", []string{"*"}, fmt.Sprintf("A list of plugins to enable. '*' enables all build-in and customized plugins, 'foo' enables the plugin named 'foo', '*,-foo' disables the plugin named 'foo'.\nAll build-in plugins: %s.", strings.Join(frameworkplugins.NewInTreeRegistry().FactoryNames(), ","))) diff --git a/cmd/scheduler/app/scheduler.go b/cmd/scheduler/app/scheduler.go index 826adbf99a0c..9d4bad259136 100644 --- a/cmd/scheduler/app/scheduler.go +++ b/cmd/scheduler/app/scheduler.go @@ -170,7 +170,7 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt scheduler.WithEnableSchedulerEstimator(opts.EnableSchedulerEstimator), scheduler.WithDisableSchedulerEstimatorInPullMode(opts.DisableSchedulerEstimatorInPullMode), scheduler.WithSchedulerEstimatorServicePrefix(opts.SchedulerEstimatorServicePrefix), - scheduler.WithSchedulerEstimatorPort(opts.SchedulerEstimatorPort), + scheduler.WithSchedulerEstimatorConnection(opts.SchedulerEstimatorPort, opts.SchedulerEstimatorCertFile, opts.SchedulerEstimatorKeyFile, opts.SchedulerEstimatorCaFile, opts.InsecureSkipEstimatorVerify), scheduler.WithSchedulerEstimatorTimeout(opts.SchedulerEstimatorTimeout), scheduler.WithEnableEmptyWorkloadPropagation(opts.EnableEmptyWorkloadPropagation), scheduler.WithEnableSchedulerPlugin(opts.Plugins), diff --git a/operator/pkg/controlplane/manifests.go b/operator/pkg/controlplane/manifests.go index e7480b91d25d..fc67c4d2fa96 100644 --- a/operator/pkg/controlplane/manifests.go +++ b/operator/pkg/controlplane/manifests.go @@ -188,6 +188,9 @@ spec: - --secure-port=10351 - --enable-scheduler-estimator=true - --leader-elect-resource-namespace={{ .SystemNamespace }} + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -199,10 +202,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/karmada/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: {{ .KarmadaCertsSecret }} - name: kubeconfig secret: secretName: {{ .KubeconfigSecret }} @@ -241,6 +250,9 @@ spec: - --kubeconfig=/etc/karmada/kubeconfig - --bind-address=0.0.0.0 - --leader-elect-resource-namespace={{ .SystemNamespace }} + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -252,10 +264,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/karmada/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: {{ .KarmadaCertsSecret }} - name: kubeconfig secret: secretName: {{ .KubeconfigSecret }} diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index f5689672fa81..f37d79db5758 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -45,6 +45,7 @@ import ( "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/fedinformer" "github.com/karmada-io/karmada/pkg/util/gclient" + "github.com/karmada-io/karmada/pkg/util/grpcconnection" ) const ( @@ -65,7 +66,7 @@ type Descheduler struct { schedulerEstimatorCache *estimatorclient.SchedulerEstimatorCache schedulerEstimatorServicePrefix string - schedulerEstimatorPort int + schedulerEstimatorClientConfig *grpcconnection.ClientConfig schedulerEstimatorWorker util.AsyncWorker unschedulableThreshold time.Duration @@ -77,15 +78,21 @@ type Descheduler struct { func NewDescheduler(karmadaClient karmadaclientset.Interface, kubeClient kubernetes.Interface, opts *options.Options) *Descheduler { factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0) desched := &Descheduler{ - KarmadaClient: karmadaClient, - KubeClient: kubeClient, - informerFactory: factory, - bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(), - bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(), - clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(), - clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(), - schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(), - schedulerEstimatorPort: opts.SchedulerEstimatorPort, + KarmadaClient: karmadaClient, + KubeClient: kubeClient, + informerFactory: factory, + bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(), + bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(), + clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(), + clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(), + schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(), + schedulerEstimatorClientConfig: &grpcconnection.ClientConfig{ + InsecureSkipServerVerify: opts.InsecureSkipEstimatorVerify, + ServerAuthCAFile: opts.SchedulerEstimatorCaFile, + CertFile: opts.SchedulerEstimatorCertFile, + KeyFile: opts.SchedulerEstimatorKeyFile, + TargetPort: opts.SchedulerEstimatorPort, + }, schedulerEstimatorServicePrefix: opts.SchedulerEstimatorServicePrefix, unschedulableThreshold: opts.UnschedulableThreshold.Duration, deschedulingInterval: opts.DeschedulingInterval.Duration, @@ -273,7 +280,7 @@ func (d *Descheduler) establishEstimatorConnections() { return } for i := range clusterList.Items { - if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort); err != nil { + if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig); err != nil { klog.Error(err) } } @@ -293,7 +300,7 @@ func (d *Descheduler) reconcileEstimatorConnection(key util.QueueKey) error { } return err } - return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort) + return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig) } func (d *Descheduler) recordDescheduleResultEventForResourceBinding(rb *workv1alpha2.ResourceBinding, message string, err error) { diff --git a/pkg/estimator/client/cache.go b/pkg/estimator/client/cache.go index 3c278e240f0d..cc7ad0f6a9c9 100644 --- a/pkg/estimator/client/cache.go +++ b/pkg/estimator/client/cache.go @@ -27,6 +27,7 @@ import ( estimatorservice "github.com/karmada-io/karmada/pkg/estimator/service" "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/grpcconnection" "github.com/karmada-io/karmada/pkg/util/names" ) @@ -96,19 +97,19 @@ func (c *SchedulerEstimatorCache) GetClient(name string) (estimatorservice.Estim } // EstablishConnection establishes a new gRPC connection with the specified cluster scheduler estimator. -func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, port int) error { +func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, grpcConfig *grpcconnection.ClientConfig) error { if estimatorCache.IsEstimatorExist(name) { return nil } serverAddr, err := resolveCluster(kubeClient, util.NamespaceKarmadaSystem, - names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(port)) + names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(grpcConfig.TargetPort)) if err != nil { return err } klog.Infof("Start dialing estimator server(%s) of cluster(%s).", serverAddr, name) - cc, err := util.Dial(serverAddr, 5*time.Second) + cc, err := grpcConfig.DialWithTimeOut(serverAddr, 5*time.Second) if err != nil { klog.Errorf("Failed to dial cluster(%s): %v.", name, err) return err diff --git a/pkg/estimator/client/service.go b/pkg/estimator/client/service.go index 323cd7ca8661..c1216557763c 100644 --- a/pkg/estimator/client/service.go +++ b/pkg/estimator/client/service.go @@ -41,7 +41,7 @@ func resolveCluster(kubeClient kubernetes.Interface, namespace, id string, port * But the Service resource is defined in Host Kubernetes Cluster. So we cannot get its content here. * The best thing we can do is just glue host:port together, and try to connect to it. */ - return net.JoinHostPort(id, fmt.Sprintf("%d", port)), nil + return net.JoinHostPort(fmt.Sprintf("%s.%s.svc.cluster.local", id, namespace), fmt.Sprintf("%d", port)), nil } return "", err diff --git a/pkg/estimator/server/server.go b/pkg/estimator/server/server.go index d8b6156c7f26..693f435b9613 100644 --- a/pkg/estimator/server/server.go +++ b/pkg/estimator/server/server.go @@ -23,7 +23,6 @@ import ( "time" "github.com/kr/pretty" - "google.golang.org/grpc" "google.golang.org/grpc/metadata" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -53,6 +52,7 @@ import ( "github.com/karmada-io/karmada/pkg/util/fedinformer" "github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager" "github.com/karmada-io/karmada/pkg/util/fedinformer/keys" + "github.com/karmada-io/karmada/pkg/util/grpcconnection" "github.com/karmada-io/karmada/pkg/util/helper" schedcache "github.com/karmada-io/karmada/pkg/util/lifted/scheduler/cache" "github.com/karmada-io/karmada/pkg/util/lifted/scheduler/framework/parallelize" @@ -73,7 +73,6 @@ var ( // AccurateSchedulerEstimatorServer is the gRPC server of a cluster accurate scheduler estimator. // Please see https://github.com/karmada-io/karmada/pull/580 (#580). type AccurateSchedulerEstimatorServer struct { - port int clusterName string kubeClient kubernetes.Interface restMapper meta.RESTMapper @@ -85,6 +84,8 @@ type AccurateSchedulerEstimatorServer struct { estimateFramework framework.Framework Cache schedcache.Cache + + GrpcConfig *grpcconnection.ServerConfig } // NewEstimatorServer creates an instance of AccurateSchedulerEstimatorServer. @@ -101,7 +102,6 @@ func NewEstimatorServer( informerFactory.InformerFor(&corev1.Pod{}, newPodInformer) es := &AccurateSchedulerEstimatorServer{ - port: opts.ServerPort, clusterName: opts.ClusterName, kubeClient: kubeClient, restMapper: restMapper, @@ -113,6 +113,13 @@ func NewEstimatorServer( }, parallelizer: parallelize.NewParallelizer(opts.Parallelism), Cache: schedcache.New(durationToExpireAssumedPod, stopChan), + GrpcConfig: &grpcconnection.ServerConfig{ + InsecureSkipClientVerify: opts.InsecureSkipGrpcClientVerify, + ClientAuthCAFile: opts.GrpcClientCaFile, + CertFile: opts.GrpcAuthCertFile, + KeyFile: opts.GrpcAuthKeyFile, + ServerPort: opts.ServerPort, + }, } // ignore the error here because the informers haven't been started _ = informerFactory.Core().V1().Nodes().Informer().SetTransform(fedinformer.StripUnusedFields) @@ -154,14 +161,17 @@ func (es *AccurateSchedulerEstimatorServer) Start(ctx context.Context) error { } // Listen a port and register the gRPC server. - l, err := net.Listen("tcp", fmt.Sprintf(":%d", es.port)) + l, err := net.Listen("tcp", fmt.Sprintf(":%d", es.GrpcConfig.ServerPort)) if err != nil { - return fmt.Errorf("failed to listen port %d: %v", es.port, err) + return fmt.Errorf("failed to listen port %d: %v", es.GrpcConfig.ServerPort, err) } - klog.Infof("Listening port: %d", es.port) + klog.Infof("Listening port: %d", es.GrpcConfig.ServerPort) defer l.Close() - s := grpc.NewServer() + s, err := es.GrpcConfig.NewServer() + if err != nil { + return fmt.Errorf("failed to create grpc server: %v", err) + } estimatorservice.RegisterEstimatorServer(s, es) // Graceful stop when the context is cancelled. diff --git a/pkg/karmadactl/addons/descheduler/manifests.go b/pkg/karmadactl/addons/descheduler/manifests.go index af1bd357c56b..711b1ebbc025 100644 --- a/pkg/karmadactl/addons/descheduler/manifests.go +++ b/pkg/karmadactl/addons/descheduler/manifests.go @@ -47,6 +47,9 @@ spec: - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 - --leader-elect-resource-namespace={{ .Namespace }} + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -58,10 +61,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: k8s-certs + secret: + secretName: karmada-cert - name: kubeconfig secret: secretName: kubeconfig diff --git a/pkg/karmadactl/addons/estimator/manifests.go b/pkg/karmadactl/addons/estimator/manifests.go index a11e29212f99..3eb58a9d977e 100644 --- a/pkg/karmadactl/addons/estimator/manifests.go +++ b/pkg/karmadactl/addons/estimator/manifests.go @@ -48,6 +48,10 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{ .MemberClusterName}}-kubeconfig - --cluster-name={{ .MemberClusterName}} + - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt + - --grpc-auth-key-file=/etc/karmada/pki/karmada.key + - --client-cert-auth=true + - --grpc-client-ca-file=/etc/karmada/pki/ca.crt livenessProbe: httpGet: path: /healthz @@ -58,10 +62,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true - name: member-kubeconfig subPath: {{ .MemberClusterName}}-kubeconfig mountPath: /etc/{{ .MemberClusterName}}-kubeconfig volumes: + - name: k8s-certs + secret: + secretName: karmada-cert - name: member-kubeconfig secret: secretName: {{ .MemberClusterName}}-kubeconfig diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index bb0ea0479e69..f7bacf7522f1 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -454,6 +454,9 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment "--secure-port=10351", "--enable-scheduler-estimator=true", "--leader-elect=true", + "--scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt", + "--scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt", + "--scheduler-estimator-key-file=/etc/karmada/pki/karmada.key", fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace), "--v=4", }, @@ -465,6 +468,11 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment MountPath: kubeConfigContainerMountPath, SubPath: KubeConfigSecretAndMountName, }, + { + Name: globaloptions.KarmadaCertsName, + ReadOnly: true, + MountPath: karmadaCertsVolumeMountPath, + }, }, }, }, @@ -477,6 +485,14 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment }, }, }, + { + Name: globaloptions.KarmadaCertsName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: globaloptions.KarmadaCertsName, + }, + }, + }, }, Tolerations: []corev1.Toleration{ { diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 2f4e7737d895..0e9f0a3dc78b 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -56,6 +56,7 @@ import ( "github.com/karmada-io/karmada/pkg/scheduler/metrics" "github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag" "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/grpcconnection" "github.com/karmada-io/karmada/pkg/util/helper" utilmetrics "github.com/karmada-io/karmada/pkg/util/metrics" ) @@ -106,8 +107,8 @@ type Scheduler struct { disableSchedulerEstimatorInPullMode bool schedulerEstimatorCache *estimatorclient.SchedulerEstimatorCache schedulerEstimatorServicePrefix string - schedulerEstimatorPort int schedulerEstimatorWorker util.AsyncWorker + schedulerEstimatorClientConfig *grpcconnection.ClientConfig schedulerName string enableEmptyWorkloadPropagation bool @@ -122,8 +123,6 @@ type schedulerOptions struct { schedulerEstimatorTimeout metav1.Duration // SchedulerEstimatorServicePrefix presents the prefix of the accurate scheduler estimator service name. schedulerEstimatorServicePrefix string - // schedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. - schedulerEstimatorPort int // schedulerName is the name of the scheduler. Default is "default-scheduler". schedulerName string //enableEmptyWorkloadPropagation represents whether allow workload with replicas 0 propagated to member clusters should be enabled @@ -134,6 +133,8 @@ type schedulerOptions struct { plugins []string // contains the options for rate limiter. RateLimiterOptions ratelimiterflag.Options + // schedulerEstimatorClientConfig contains the configuration of GRPC. + schedulerEstimatorClientConfig *grpcconnection.ClientConfig } // Option configures a Scheduler @@ -146,6 +147,19 @@ func WithEnableSchedulerEstimator(enableSchedulerEstimator bool) Option { } } +// WithSchedulerEstimatorConnection sets the grpc config for scheduler +func WithSchedulerEstimatorConnection(port int, certFile, keyFile, trustedCAFile string, insecureSkipVerify bool) Option { + return func(o *schedulerOptions) { + o.schedulerEstimatorClientConfig = &grpcconnection.ClientConfig{ + CertFile: certFile, + KeyFile: keyFile, + ServerAuthCAFile: trustedCAFile, + InsecureSkipServerVerify: insecureSkipVerify, + TargetPort: port, + } + } +} + // WithDisableSchedulerEstimatorInPullMode sets the disableSchedulerEstimatorInPullMode for scheduler func WithDisableSchedulerEstimatorInPullMode(disableSchedulerEstimatorInPullMode bool) Option { return func(o *schedulerOptions) { @@ -167,13 +181,6 @@ func WithSchedulerEstimatorServicePrefix(schedulerEstimatorServicePrefix string) } } -// WithSchedulerEstimatorPort sets the schedulerEstimatorPort for scheduler -func WithSchedulerEstimatorPort(schedulerEstimatorPort int) Option { - return func(o *schedulerOptions) { - o.schedulerEstimatorPort = schedulerEstimatorPort - } -} - // WithSchedulerName sets the schedulerName for scheduler func WithSchedulerName(schedulerName string) Option { return func(o *schedulerOptions) { @@ -255,7 +262,7 @@ func NewScheduler(dynamicClient dynamic.Interface, karmadaClient karmadaclientse sched.enableSchedulerEstimator = options.enableSchedulerEstimator sched.disableSchedulerEstimatorInPullMode = options.disableSchedulerEstimatorInPullMode sched.schedulerEstimatorServicePrefix = options.schedulerEstimatorServicePrefix - sched.schedulerEstimatorPort = options.schedulerEstimatorPort + sched.schedulerEstimatorClientConfig = options.schedulerEstimatorClientConfig sched.schedulerEstimatorCache = estimatorclient.NewSchedulerEstimatorCache() schedulerEstimatorWorkerOptions := util.Options{ Name: "scheduler-estimator", @@ -769,7 +776,7 @@ func (s *Scheduler) reconcileEstimatorConnection(key util.QueueKey) error { return nil } - return estimatorclient.EstablishConnection(s.KubeClient, name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorPort) + return estimatorclient.EstablishConnection(s.KubeClient, name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorClientConfig) } func (s *Scheduler) establishEstimatorConnections() { @@ -782,7 +789,7 @@ func (s *Scheduler) establishEstimatorConnections() { if clusterList.Items[i].Spec.SyncMode == clusterv1alpha1.Pull && s.disableSchedulerEstimatorInPullMode { continue } - if err = estimatorclient.EstablishConnection(s.KubeClient, clusterList.Items[i].Name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorPort); err != nil { + if err = estimatorclient.EstablishConnection(s.KubeClient, clusterList.Items[i].Name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorClientConfig); err != nil { klog.Error(err) } } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 584eb226e224..da83e28fec2b 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -53,16 +53,16 @@ func TestCreateScheduler(t *testing.T) { name: "scheduler with enableSchedulerEstimator enabled", opts: []Option{ WithEnableSchedulerEstimator(true), - WithSchedulerEstimatorPort(port), + WithSchedulerEstimatorConnection(port, "", "", "", false), }, enableSchedulerEstimator: true, schedulerEstimatorPort: port, }, { - name: "scheduler with enableSchedulerEstimator disabled, WithSchedulerEstimatorPort enabled", + name: "scheduler with enableSchedulerEstimator disabled, WithSchedulerEstimatorConnection enabled", opts: []Option{ WithEnableSchedulerEstimator(false), - WithSchedulerEstimatorPort(port), + WithSchedulerEstimatorConnection(port, "", "", "", false), }, enableSchedulerEstimator: false, }, @@ -79,8 +79,8 @@ func TestCreateScheduler(t *testing.T) { t.Errorf("unexpected enableSchedulerEstimator want %v, got %v", tc.enableSchedulerEstimator, sche.enableSchedulerEstimator) } - if tc.schedulerEstimatorPort != sche.schedulerEstimatorPort { - t.Errorf("unexpected schedulerEstimatorPort want %v, got %v", tc.schedulerEstimatorPort, sche.schedulerEstimatorPort) + if tc.enableSchedulerEstimator && tc.schedulerEstimatorPort != sche.schedulerEstimatorClientConfig.TargetPort { + t.Errorf("unexpected schedulerEstimatorPort want %v, got %v", tc.schedulerEstimatorPort, sche.schedulerEstimatorClientConfig.TargetPort) } }) } diff --git a/pkg/util/dial.go b/pkg/util/dial.go deleted file mode 100644 index ab12bcda5ee1..000000000000 --- a/pkg/util/dial.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2021 The Karmada Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "context" - "fmt" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -// Dial establishes the gRPC communication. -func Dial(path string, timeout time.Duration) (*grpc.ClientConn, error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - opts := []grpc.DialOption{ - grpc.WithBlock(), - grpc.WithTransportCredentials(insecure.NewCredentials()), - } - cc, err := grpc.DialContext(ctx, path, opts...) - if err != nil { - return nil, fmt.Errorf("dial %s error: %v", path, err) - } - - return cc, nil -} diff --git a/pkg/util/grpcconnection/config.go b/pkg/util/grpcconnection/config.go new file mode 100644 index 000000000000..149d0aeefa4e --- /dev/null +++ b/pkg/util/grpcconnection/config.go @@ -0,0 +1,147 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package grpcconnection + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "time" + + "google.golang.org/grpc" + grpccredentials "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +// ServerConfig the config of GRPC server side. +type ServerConfig struct { + // ServerPort The secure port on which to serve gRPC. + ServerPort int + // InsecureSkipClientVerify Controls whether verifies the client's certificate chain and host name. + // When this is set to false, server will check all incoming HTTPS requests for a client certificate signed by the trusted CA, + // requests that don’t supply a valid client certificate will fail. If authentication is enabled, + // the certificate provides credentials for the user name given by the Common Name field. + InsecureSkipClientVerify bool + // ClientAuthCAFile SSL Certificate Authority file used to verify grpc client certificates on incoming requests. + ClientAuthCAFile string + // CertFile SSL certification file used for grpc SSL/TLS connections. + CertFile string + // KeyFile SSL key file used for grpc SSL/TLS connections. + KeyFile string +} + +// ClientConfig the config of GRPC client side. +type ClientConfig struct { + // TargetPort the target port on which to establish a gRPC connection. + TargetPort int + // InsecureSkipServerVerify controls whether a client verifies the server's + // certificate chain and host name. If InsecureSkipServerVerify is true, crypto/tls + // accepts any certificate presented by the server and any host name in that + // certificate. In this mode, TLS is susceptible to machine-in-the-middle + // attacks unless custom verification is used. This should be used only for + // testing or in combination with VerifyConnection or VerifyPeerCertificate. + InsecureSkipServerVerify bool + // ServerAuthCAFile SSL Certificate Authority file used to verify grpc server certificates. + ServerAuthCAFile string + // SSL certification file used for grpc SSL/TLS connections. + CertFile string + // SSL key file used for grpc SSL/TLS connections. + KeyFile string +} + +// NewServer creates a gRPC server which has no service registered and has not +// started to accept requests yet. +func (s *ServerConfig) NewServer() (*grpc.Server, error) { + if s.CertFile == "" || s.KeyFile == "" { + return grpc.NewServer(), nil + } + + cert, err := tls.LoadX509KeyPair(s.CertFile, s.KeyFile) + if err != nil { + return nil, err + } + config := &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS13, + } + + if s.ClientAuthCAFile != "" { + certPool := x509.NewCertPool() + ca, err := os.ReadFile(s.ClientAuthCAFile) + if err != nil { + return nil, err + } + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return nil, fmt.Errorf("failed to append ca into certPool") + } + config.ClientCAs = certPool + if !s.InsecureSkipClientVerify { + config.ClientAuth = tls.RequireAndVerifyClientCert + } + } + + return grpc.NewServer(grpc.Creds(grpccredentials.NewTLS(config))), nil +} + +// DialWithTimeOut creates a client connection to the given target. +func (c *ClientConfig) DialWithTimeOut(path string, timeout time.Duration) (*grpc.ClientConn, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + opts := []grpc.DialOption{ + grpc.WithBlock(), + } + + var cred grpccredentials.TransportCredentials + if c.ServerAuthCAFile == "" && !c.InsecureSkipServerVerify { + // insecure connection + cred = insecure.NewCredentials() + } else { + // server-side TLS + config := &tls.Config{InsecureSkipVerify: c.InsecureSkipServerVerify} // nolint:gosec // G402: TLS InsecureSkipEstimatorVerify may be true. + if c.ServerAuthCAFile != "" { + certPool := x509.NewCertPool() + ca, err := os.ReadFile(c.ServerAuthCAFile) + if err != nil { + return nil, err + } + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return nil, fmt.Errorf("failed to append ca certs") + } + config.RootCAs = certPool + } + if c.CertFile != "" && c.KeyFile != "" { + // mutual TLS + certificate, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) + if err != nil { + return nil, err + } + config.Certificates = []tls.Certificate{certificate} + } + cred = grpccredentials.NewTLS(config) + } + + opts = append(opts, grpc.WithTransportCredentials(cred)) + cc, err := grpc.DialContext(ctx, path, opts...) + if err != nil { + return nil, fmt.Errorf("dial %s error: %v", path, err) + } + + return cc, nil +}