From 58bb1a41eb672edca7f77309346c1daf9735de08 Mon Sep 17 00:00:00 2001 From: zhouhaoA1 Date: Thu, 14 Mar 2024 14:39:53 +0800 Subject: [PATCH] fix: update projected Signed-off-by: renxiangyu --- .../cluster-manager/app/manager.go | 34 ++-- .../controllers/pod/root_pod_controller.go | 165 ++++++++++++------ .../promote/detach/pod_detach_action.go | 1 + 3 files changed, 126 insertions(+), 74 deletions(-) diff --git a/cmd/clustertree/cluster-manager/app/manager.go b/cmd/clustertree/cluster-manager/app/manager.go index ba085dd52..c9fa011db 100644 --- a/cmd/clustertree/cluster-manager/app/manager.go +++ b/cmd/clustertree/cluster-manager/app/manager.go @@ -3,6 +3,7 @@ package app import ( "context" "fmt" + "k8s.io/client-go/discovery" "os" "github.com/spf13/cobra" @@ -22,6 +23,7 @@ import ( "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers" "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/mcs" podcontrollers "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pod" + "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/promote" "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pv" "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pvc" nodeserver "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/node-server" @@ -161,11 +163,11 @@ func run(ctx context.Context, opts *options.Options) error { return err } - //discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) - //if err != nil { - // klog.Errorf("Unable to create discoveryClient: %v", err) - // return err - //} + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + klog.Errorf("Unable to create discoveryClient: %v", err) + return err + } // add cluster controller clusterController := clusterManager.ClusterController{ @@ -268,17 +270,17 @@ func run(ctx context.Context, opts *options.Options) error { } } - //promotePolicyController := promote.PromotePolicyController{ - // RootClient: mgr.GetClient(), - // RootClientSet: rootClient, - // RootDynamicClient: dynamicClient, - // RootDiscoveryClient: discoveryClient, - // GlobalLeafManager: globalleafManager, - // PromotePolicyOptions: opts.PromotePolicyOptions, - //} - //if err = promotePolicyController.SetupWithManager(mgr); err != nil { - // return fmt.Errorf("error starting %s: %v", promote.PromotePolicyControllerName, err) - //} + promotePolicyController := promote.PromotePolicyController{ + RootClient: mgr.GetClient(), + RootClientSet: rootClient, + RootDynamicClient: dynamicClient, + RootDiscoveryClient: discoveryClient, + GlobalLeafManager: globalleafManager, + PromotePolicyOptions: opts.PromotePolicyOptions, + } + if err = promotePolicyController.SetupWithManager(mgr); err != nil { + return fmt.Errorf("error starting %s: %v", promote.PromotePolicyControllerName, err) + } // init commonController for i, gvr := range controllers.SYNC_GVRS { diff --git a/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go b/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go index f76d87c46..9a62939f8 100644 --- a/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go +++ b/pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go @@ -560,73 +560,122 @@ func (r *RootPodReconciler) changeToMasterCoreDNS(ctx context.Context, pod *core } } -// projectedHandler Process the project volume, creating and mounting secret, configmap, DownwardAPI, +// createProjectedHandler Process the project volume, creating and mounting secret, configmap, DownwardAPI, // and ServiceAccountToken from the project volume in the member cluster to the pod of the host cluster -func (r *RootPodReconciler) projectedHandler(ctx context.Context, lr *leafUtils.LeafResource, pod *corev1.Pod) { +func (r *RootPodReconciler) createProjectedHandler(ctx context.Context, lr *leafUtils.LeafResource, pod *corev1.Pod) { if pod.Spec.Volumes == nil { return } for _, volume := range pod.Spec.Volumes { if volume.Projected != nil { - falseValue := false - pod.Spec.AutomountServiceAccountToken = &falseValue - - saName := pod.Spec.ServiceAccountName - var sources []corev1.VolumeProjection - - for _, projectedVolumeSource := range volume.Projected.Sources { - // Process all resources for the rootpod - if projectedVolumeSource.ServiceAccountToken != nil { - tokenSecretName, err := r.createSATokenInLeafCluster(ctx, lr, saName, pod) - if err != nil { - klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s, err: %s", pod.Namespace, pod.Name, err) - return - } - secretProjection := corev1.VolumeProjection{ - Secret: &corev1.SecretProjection{ - Items: []corev1.KeyToPath{ - { - Key: "token", - Path: projectedVolumeSource.ServiceAccountToken.Path, - }, - }, - }, - } - secretProjection.Secret.Name = tokenSecretName - sources = append(sources, secretProjection) - } - if projectedVolumeSource.ConfigMap != nil { - cmName, err := r.createConfigMapInLeafCluster(ctx, lr, projectedVolumeSource.ConfigMap.Name, pod) - if err != nil { - klog.Errorf("[convertAuth] create configmap failed, ns: %s, cm: %s, err: %s", pod.Namespace, cmName, err) - return - } - cmDeepCopy := projectedVolumeSource.DeepCopy() - cmDeepCopy.ConfigMap.Name = cmName - sources = append(sources, *cmDeepCopy) - } - if projectedVolumeSource.Secret != nil { - Secret := projectedVolumeSource.Secret - seName, err := r.createSecretInLeafCluster(ctx, lr, Secret.Name, pod) - if err != nil { - klog.Errorf("[convertAuth] create secret failed, ns: %s, cm: %s, err: %s", pod.Namespace, seName, err) - return - } - secretDeepCopy := projectedVolumeSource.DeepCopy() - secretDeepCopy.Secret.Name = seName - sources = append(sources, *secretDeepCopy) - } - if projectedVolumeSource.DownwardAPI != nil { - DownwardAPIProjection := corev1.VolumeProjection{ - DownwardAPI: projectedVolumeSource.DownwardAPI, - } - sources = append(sources, DownwardAPIProjection) + if sources := r.projectedHandler(ctx, lr, volume, pod); sources != nil { + volume.Projected.Sources = sources + } + } + } +} + +// updateProjectedHandler update projected volume +func (r *RootPodReconciler) updateProjectedHandler(ctx context.Context, lr *leafUtils.LeafResource, rootPod, podCopy *corev1.Pod) { + if rootPod.Spec.Volumes == nil { + return + } + var leafPodVolumes []corev1.Volume + if podCopy.Spec.Volumes == nil { + leafPodVolumes = nil + } else { + leafPodVolumes = podCopy.Spec.Volumes + } + + var volumeCopy []corev1.Volume + + for _, volume := range rootPod.Spec.Volumes { + if volume.Projected != nil { + if _, flag := findVolumeInClient(volume, leafPodVolumes); !flag { + if sources := r.projectedHandler(ctx, lr, volume, podCopy); sources != nil { + volume.Projected.Sources = sources } } - volume.Projected.Sources = sources + } + volumeCopy = append(volumeCopy, volume) + } + podCopy.Spec.Volumes = volumeCopy +} + +func (r *RootPodReconciler) projectedHandler(ctx context.Context, lr *leafUtils.LeafResource, volume corev1.Volume, pod *corev1.Pod) []corev1.VolumeProjection { + falseValue := false + pod.Spec.AutomountServiceAccountToken = &falseValue + + saName := pod.Spec.ServiceAccountName + var sources []corev1.VolumeProjection + + for _, projectedVolumeSource := range volume.Projected.Sources { + // Process all resources for the rootpod + if projectedVolumeSource.ServiceAccountToken != nil { + tokenSecretName, err := r.createSATokenInLeafCluster(ctx, lr, saName, pod) + if err != nil { + klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s, err: %s", pod.Namespace, pod.Name, err) + return nil + } + secretProjection := corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + Items: []corev1.KeyToPath{ + { + Key: "token", + Path: projectedVolumeSource.ServiceAccountToken.Path, + }, + }, + }, + } + secretProjection.Secret.Name = tokenSecretName + sources = append(sources, secretProjection) + } + if projectedVolumeSource.ConfigMap != nil { + cmName, err := r.createConfigMapInLeafCluster(ctx, lr, projectedVolumeSource.ConfigMap.Name, pod) + if err != nil { + klog.Errorf("[convertAuth] create configmap failed, ns: %s, cm: %s, err: %s", pod.Namespace, cmName, err) + return nil + } + cmDeepCopy := projectedVolumeSource.DeepCopy() + cmDeepCopy.ConfigMap.Name = cmName + sources = append(sources, *cmDeepCopy) + } + if projectedVolumeSource.Secret != nil { + Secret := projectedVolumeSource.Secret + seName, err := r.createSecretInLeafCluster(ctx, lr, Secret.Name, pod) + if err != nil { + klog.Errorf("[convertAuth] create secret failed, ns: %s, cm: %s, err: %s", pod.Namespace, seName, err) + return nil + } + secretDeepCopy := projectedVolumeSource.DeepCopy() + secretDeepCopy.Secret.Name = seName + sources = append(sources, *secretDeepCopy) + } + if projectedVolumeSource.DownwardAPI != nil { + DownwardAPIProjection := corev1.VolumeProjection{ + DownwardAPI: projectedVolumeSource.DownwardAPI, + } + sources = append(sources, DownwardAPIProjection) + } + } + return sources +} + +func findVolumeInClient(volumeInRoot corev1.Volume, volumes []corev1.Volume) (corev1.Volume, bool) { + if volumes == nil { + return corev1.Volume{}, false + } + + for _, volume := range volumes { + if volume.Projected != nil && volume.Name == volumeInRoot.Name { + if reflect.DeepEqual(volume.Projected, volumeInRoot.Projected) { + return volume, true + } } } + + return corev1.Volume{}, false } // createServiceAccountInLeafCluster Create an sa corresponding to token-secret in member cluster @@ -882,7 +931,7 @@ func (r *RootPodReconciler) CreatePodInLeafCluster(ctx context.Context, lr *leaf klog.V(4).Infof("Creating Volumes successed %+v", basicPod) } - r.projectedHandler(ctx, lr, basicPod) + r.createProjectedHandler(ctx, lr, basicPod) if !r.Options.MultiClusterService { r.changeToMasterCoreDNS(ctx, basicPod, r.Options) @@ -924,7 +973,7 @@ func (r *RootPodReconciler) UpdatePodInLeafCluster(ctx context.Context, lr *leaf return nil } - r.projectedHandler(ctx, lr, podCopy) + r.updateProjectedHandler(ctx, lr, rootPod, podCopy) if !r.Options.MultiClusterService { r.changeToMasterCoreDNS(ctx, podCopy, r.Options) diff --git a/pkg/clustertree/cluster-manager/controllers/promote/detach/pod_detach_action.go b/pkg/clustertree/cluster-manager/controllers/promote/detach/pod_detach_action.go index 9a75ff698..5cf0ce8b3 100644 --- a/pkg/clustertree/cluster-manager/controllers/promote/detach/pod_detach_action.go +++ b/pkg/clustertree/cluster-manager/controllers/promote/detach/pod_detach_action.go @@ -75,6 +75,7 @@ func (p *PodAction) Revert(obj *unstructured.Unstructured, client client.Dynamic if labels != nil { if _, ok := labels["kosmos-io/pod"]; ok { delete(labels, "kosmos-io/pod") + delete(labels, "kosmos-io/synced") updatedPod.SetLabels(labels) podMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&updatedPod) if err != nil {