Skip to content

Commit

Permalink
fix: update projected
Browse files Browse the repository at this point in the history 
Signed-off-by: renxiangyu <renxiangyu_yewu@cmss.chinamobile.com
Signed-off-by: zhouhaoA1 <zhouhao_yewu@cmss.chinamobile.com>
  • Loading branch information
@zhouhaoA1
zhouhaoA1 committed Mar 18, 2024
1 parent 7c3eeda commit 923bddf
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 74 deletions.
34 changes: 18 additions & 16 deletions cmd/clustertree/cluster-manager/app/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
Expand All @@ -22,6 +23,7 @@ import (
"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers"
"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/mcs"
podcontrollers "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pod"
"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/promote"
"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pv"
"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pvc"
nodeserver "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/node-server"
Expand Down Expand Up @@ -161,11 +163,11 @@ func run(ctx context.Context, opts *options.Options) error {
return err
}

//discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
//if err != nil {
// klog.Errorf("Unable to create discoveryClient: %v", err)
// return err
//}
discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
if err != nil {
klog.Errorf("Unable to create discoveryClient: %v", err)
return err
}

// add cluster controller
clusterController := clusterManager.ClusterController{
Expand Down Expand Up @@ -268,17 +270,17 @@ func run(ctx context.Context, opts *options.Options) error {
}
}

//promotePolicyController := promote.PromotePolicyController{
// RootClient: mgr.GetClient(),
// RootClientSet: rootClient,
// RootDynamicClient: dynamicClient,
// RootDiscoveryClient: discoveryClient,
// GlobalLeafManager: globalleafManager,
// PromotePolicyOptions: opts.PromotePolicyOptions,
//}
//if err = promotePolicyController.SetupWithManager(mgr); err != nil {
// return fmt.Errorf("error starting %s: %v", promote.PromotePolicyControllerName, err)
//}
promotePolicyController := promote.PromotePolicyController{
RootClient: mgr.GetClient(),
RootClientSet: rootClient,
RootDynamicClient: dynamicClient,
RootDiscoveryClient: discoveryClient,
GlobalLeafManager: globalleafManager,
PromotePolicyOptions: opts.PromotePolicyOptions,
}
if err = promotePolicyController.SetupWithManager(mgr); err != nil {
return fmt.Errorf("error starting %s: %v", promote.PromotePolicyControllerName, err)
}

// init commonController
for i, gvr := range controllers.SYNC_GVRS {
Expand Down
165 changes: 107 additions & 58 deletions pkg/clustertree/cluster-manager/controllers/pod/root_pod_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -560,73 +560,122 @@ func (r *RootPodReconciler) changeToMasterCoreDNS(ctx context.Context, pod *core
}
}

// projectedHandler Process the project volume, creating and mounting secret, configmap, DownwardAPI,
// createProjectedHandler Process the project volume, creating and mounting secret, configmap, DownwardAPI,
// and ServiceAccountToken from the project volume in the member cluster to the pod of the host cluster
func (r *RootPodReconciler) projectedHandler(ctx context.Context, lr *leafUtils.LeafResource, pod *corev1.Pod) {
func (r *RootPodReconciler) createProjectedHandler(ctx context.Context, lr *leafUtils.LeafResource, pod *corev1.Pod) {
if pod.Spec.Volumes == nil {
return
}

for _, volume := range pod.Spec.Volumes {
if volume.Projected != nil {
falseValue := false
pod.Spec.AutomountServiceAccountToken = &falseValue

saName := pod.Spec.ServiceAccountName
var sources []corev1.VolumeProjection

for _, projectedVolumeSource := range volume.Projected.Sources {
// Process all resources for the rootpod
if projectedVolumeSource.ServiceAccountToken != nil {
tokenSecretName, err := r.createSATokenInLeafCluster(ctx, lr, saName, pod)
if err != nil {
klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s, err: %s", pod.Namespace, pod.Name, err)
return
}
secretProjection := corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
Items: []corev1.KeyToPath{
{
Key: "token",
Path: projectedVolumeSource.ServiceAccountToken.Path,
},
},
},
}
secretProjection.Secret.Name = tokenSecretName
sources = append(sources, secretProjection)
}
if projectedVolumeSource.ConfigMap != nil {
cmName, err := r.createConfigMapInLeafCluster(ctx, lr, projectedVolumeSource.ConfigMap.Name, pod)
if err != nil {
klog.Errorf("[convertAuth] create configmap failed, ns: %s, cm: %s, err: %s", pod.Namespace, cmName, err)
return
}
cmDeepCopy := projectedVolumeSource.DeepCopy()
cmDeepCopy.ConfigMap.Name = cmName
sources = append(sources, *cmDeepCopy)
}
if projectedVolumeSource.Secret != nil {
Secret := projectedVolumeSource.Secret
seName, err := r.createSecretInLeafCluster(ctx, lr, Secret.Name, pod)
if err != nil {
klog.Errorf("[convertAuth] create secret failed, ns: %s, cm: %s, err: %s", pod.Namespace, seName, err)
return
}
secretDeepCopy := projectedVolumeSource.DeepCopy()
secretDeepCopy.Secret.Name = seName
sources = append(sources, *secretDeepCopy)
}
if projectedVolumeSource.DownwardAPI != nil {
DownwardAPIProjection := corev1.VolumeProjection{
DownwardAPI: projectedVolumeSource.DownwardAPI,
}
sources = append(sources, DownwardAPIProjection)
if sources := r.projectedHandler(ctx, lr, volume, pod); sources != nil {
volume.Projected.Sources = sources
}
}
}
}

// updateProjectedHandler update projected volume
func (r *RootPodReconciler) updateProjectedHandler(ctx context.Context, lr *leafUtils.LeafResource, rootPod, podCopy *corev1.Pod) {
if rootPod.Spec.Volumes == nil {
return
}
var leafPodVolumes []corev1.Volume
if podCopy.Spec.Volumes == nil {
leafPodVolumes = nil
} else {
leafPodVolumes = podCopy.Spec.Volumes
}

var volumeCopy []corev1.Volume

for _, volume := range rootPod.Spec.Volumes {
if volume.Projected != nil {
if _, flag := findVolumeInClient(volume, leafPodVolumes); !flag {
if sources := r.projectedHandler(ctx, lr, volume, podCopy); sources != nil {
volume.Projected.Sources = sources
}
}
volume.Projected.Sources = sources
}
volumeCopy = append(volumeCopy, volume)
}
podCopy.Spec.Volumes = volumeCopy
}

func (r *RootPodReconciler) projectedHandler(ctx context.Context, lr *leafUtils.LeafResource, volume corev1.Volume, pod *corev1.Pod) []corev1.VolumeProjection {
falseValue := false
pod.Spec.AutomountServiceAccountToken = &falseValue

saName := pod.Spec.ServiceAccountName
var sources []corev1.VolumeProjection

for _, projectedVolumeSource := range volume.Projected.Sources {
// Process all resources for the rootpod
if projectedVolumeSource.ServiceAccountToken != nil {
tokenSecretName, err := r.createSATokenInLeafCluster(ctx, lr, saName, pod)
if err != nil {
klog.Errorf("[convertAuth] create sa secret failed, ns: %s, pod: %s, err: %s", pod.Namespace, pod.Name, err)
return nil
}
secretProjection := corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
Items: []corev1.KeyToPath{
{
Key: "token",
Path: projectedVolumeSource.ServiceAccountToken.Path,
},
},
},
}
secretProjection.Secret.Name = tokenSecretName
sources = append(sources, secretProjection)
}
if projectedVolumeSource.ConfigMap != nil {
cmName, err := r.createConfigMapInLeafCluster(ctx, lr, projectedVolumeSource.ConfigMap.Name, pod)
if err != nil {
klog.Errorf("[convertAuth] create configmap failed, ns: %s, cm: %s, err: %s", pod.Namespace, cmName, err)
return nil
}
cmDeepCopy := projectedVolumeSource.DeepCopy()
cmDeepCopy.ConfigMap.Name = cmName
sources = append(sources, *cmDeepCopy)
}
if projectedVolumeSource.Secret != nil {
Secret := projectedVolumeSource.Secret
seName, err := r.createSecretInLeafCluster(ctx, lr, Secret.Name, pod)
if err != nil {
klog.Errorf("[convertAuth] create secret failed, ns: %s, cm: %s, err: %s", pod.Namespace, seName, err)
return nil
}
secretDeepCopy := projectedVolumeSource.DeepCopy()
secretDeepCopy.Secret.Name = seName
sources = append(sources, *secretDeepCopy)
}
if projectedVolumeSource.DownwardAPI != nil {
DownwardAPIProjection := corev1.VolumeProjection{
DownwardAPI: projectedVolumeSource.DownwardAPI,
}
sources = append(sources, DownwardAPIProjection)
}
}
return sources
}

func findVolumeInClient(volumeInRoot corev1.Volume, volumes []corev1.Volume) (corev1.Volume, bool) {
if volumes == nil {
return corev1.Volume{}, false
}

for _, volume := range volumes {
if volume.Projected != nil && volume.Name == volumeInRoot.Name {
if reflect.DeepEqual(volume.Projected, volumeInRoot.Projected) {
return volume, true
}
}
}

return corev1.Volume{}, false
}

// createServiceAccountInLeafCluster Create an sa corresponding to token-secret in member cluster
Expand Down Expand Up @@ -882,7 +931,7 @@ func (r *RootPodReconciler) CreatePodInLeafCluster(ctx context.Context, lr *leaf
klog.V(4).Infof("Creating Volumes successed %+v", basicPod)
}

r.projectedHandler(ctx, lr, basicPod)
r.createProjectedHandler(ctx, lr, basicPod)

if !r.Options.MultiClusterService {
r.changeToMasterCoreDNS(ctx, basicPod, r.Options)
Expand Down Expand Up @@ -924,7 +973,7 @@ func (r *RootPodReconciler) UpdatePodInLeafCluster(ctx context.Context, lr *leaf
return nil
}

r.projectedHandler(ctx, lr, podCopy)
r.updateProjectedHandler(ctx, lr, rootPod, podCopy)

if !r.Options.MultiClusterService {
r.changeToMasterCoreDNS(ctx, podCopy, r.Options)
Expand Down
1 change: 1 addition & 0 deletions pkg/clustertree/cluster-manager/controllers/promote/detach/pod_detach_action.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ func (p *PodAction) Revert(obj *unstructured.Unstructured, client client.Dynamic
if labels != nil {
if _, ok := labels["kosmos-io/pod"]; ok {
delete(labels, "kosmos-io/pod")
delete(labels, "kosmos-io/synced")
updatedPod.SetLabels(labels)
podMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&updatedPod)
if err != nil {
Expand Down

0 comments on commit 923bddf

Please sign in to comment.