Merge pull request #8504 from Lyndon-Li/linux-windows-hybrid-deploy

Linux windows hybrid deploy

changelogs/unreleased/8504-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #8416, #8417, deploy Velero server and node-agent in linux/Windows hybrid env

pkg/cmd/cli/install/install.go

@@ -66,6 +66,7 @@ type Options struct {
 	BackupStorageConfig             flag.Map
 	VolumeSnapshotConfig            flag.Map
 	UseNodeAgent                    bool
+	UseNodeAgentWindows             bool
 	PrivilegedNodeAgent             bool
 	Wait                            bool
 	UseVolumeSnapshots              bool
@@ -117,7 +118,8 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) {
 	flags.BoolVar(&o.UseVolumeSnapshots, "use-volume-snapshots", o.UseVolumeSnapshots, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.")
 	flags.BoolVar(&o.RestoreOnly, "restore-only", o.RestoreOnly, "Run the server in restore-only mode. Optional.")
 	flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Generate resources, but don't send them to the cluster. Use with -o. Optional.")
-	flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts Velero modules that need to run in one or more nodes(i.e. Restic, Kopia).")
+	flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts and associates Velero modules that need to run in one or more Linux nodes.")
+	flags.BoolVar(&o.UseNodeAgentWindows, "use-node-agent-windows", o.UseNodeAgentWindows, "Create Velero node-agent-windows daemonset. Optional. Velero node-agent-windows hosts and associates Velero modules that need to run in one or more Windows nodes.")
 	flags.BoolVar(&o.PrivilegedNodeAgent, "privileged-node-agent", o.PrivilegedNodeAgent, "Use privileged mode for the node agent. Optional. Required to backup block devices.")
 	flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.")
 	flags.DurationVar(&o.DefaultRepoMaintenanceFrequency, "default-repo-maintain-frequency", o.DefaultRepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default. Optional.")
@@ -267,6 +269,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) {
 		SecretData:                      secretData,
 		RestoreOnly:                     o.RestoreOnly,
 		UseNodeAgent:                    o.UseNodeAgent,
+		UseNodeAgentWindows:             o.UseNodeAgentWindows,
 		PrivilegedNodeAgent:             o.PrivilegedNodeAgent,
 		UseVolumeSnapshots:              o.UseVolumeSnapshots,
 		BSLConfig:                       o.BackupStorageConfig.Data(),
@@ -392,7 +395,12 @@ func (o *Options) Run(c *cobra.Command, f client.Factory) error {
 
 		if o.UseNodeAgent {
 			fmt.Println("Waiting for node-agent daemonset to be ready.")
-			if _, err = install.DaemonSetIsReady(dynamicFactory, o.Namespace); err != nil {
+			if _, err = install.NodeAgentIsReady(dynamicFactory, o.Namespace); err != nil {
+				return errors.Wrap(err, errorMsg)
+			}
+
+			fmt.Println("Waiting for node-agent-windows daemonset to be ready.")
+			if _, err = install.NodeAgentWindowsIsReady(dynamicFactory, o.Namespace); err != nil {
 				return errors.Wrap(err, errorMsg)
 			}
 		}

pkg/cmd/cli/nodeagent/server.go

@@ -200,13 +200,31 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi
 			},
 		},
 	}
-	mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{
-		Scheme: scheme,
-		Cache:  cacheOption,
-	})
+
+	var mgr manager.Manager
+	retry := 10
+	for {
+		mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{
+			Scheme: scheme,
+			Cache:  cacheOption,
+		})
+		if err == nil {
+			break
+		}
+
+		retry--
+		if retry == 0 {
+			break
+		}
+
+		logger.WithError(err).Warn("Failed to create controller manager, need retry")
+
+		time.Sleep(time.Second)
+	}
+
 	if err != nil {
 		cancelFunc()
-		return nil, err
+		return nil, errors.Wrap(err, "error creating controller manager")
 	}
 
 	s := &nodeAgentServer{

pkg/cmd/server/server.go

@@ -239,17 +239,34 @@ func newServer(f client.Factory, config *config.Config, logger *logrus.Logger) (
 
 	ctrl.SetLogger(logrusr.New(logger))
 
-	mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{
-		Scheme: scheme,
-		Cache: cache.Options{
-			DefaultNamespaces: map[string]cache.Config{
-				f.Namespace(): {},
+	var mgr manager.Manager
+	retry := 10
+	for {
+		mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{
+			Scheme: scheme,
+			Cache: cache.Options{
+				DefaultNamespaces: map[string]cache.Config{
+					f.Namespace(): {},
+				},
 			},
-		},
-	})
+		})
+		if err == nil {
+			break
+		}
+
+		retry--
+		if retry == 0 {
+			break
+		}
+
+		logger.WithError(err).Warn("Failed to create controller manager, need retry")
+
+		time.Sleep(time.Second)
+	}
+
 	if err != nil {
 		cancelFunc()
-		return nil, err
+		return nil, errors.Wrap(err, "error creating controller manager")
 	}
 
 	credentialFileStore, err := credentials.NewNamespacedFileStore(

pkg/exposer/csi_snapshot_test.go

@@ -1146,7 +1146,7 @@ func Test_csiSnapshotExposer_DiagnoseExpose(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: velerov1.DefaultNamespace,
 			Name:      "node-agent-pod-1",
-			Labels:    map[string]string{"name": "node-agent"},
+			Labels:    map[string]string{"role": "node-agent"},
 		},
 		Spec: corev1.PodSpec{
 			NodeName: "fake-node",

pkg/exposer/generic_restore_test.go

@@ -627,7 +627,7 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: velerov1.DefaultNamespace,
 			Name:      "node-agent-pod-1",
-			Labels:    map[string]string{"name": "node-agent"},
+			Labels:    map[string]string{"role": "node-agent"},
 		},
 		Spec: corev1.PodSpec{
 			NodeName: "fake-node",

pkg/install/daemonset.go

@@ -57,22 +57,28 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet {
 	userID := int64(0)
 	mountPropagationMode := corev1.MountPropagationHostToContainer
 
+	dsName := "node-agent"
+	if c.forWindows {
+		dsName = "node-agent-windows"
+	}
+
 	daemonSet := &appsv1.DaemonSet{
-		ObjectMeta: objectMeta(namespace, "node-agent"),
+		ObjectMeta: objectMeta(namespace, dsName),
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "DaemonSet",
 			APIVersion: appsv1.SchemeGroupVersion.String(),
 		},
 		Spec: appsv1.DaemonSetSpec{
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{
-					"name": "node-agent",
+					"name": dsName,
 				},
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: podLabels(c.labels, map[string]string{
-						"name": "node-agent",
+						"name": dsName,
+						"role": "node-agent",
 					}),
 					Annotations: c.annotations,
 				},
@@ -107,7 +113,7 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet {
 					},
 					Containers: []corev1.Container{
 						{
-							Name:            "node-agent",
+							Name:            dsName,
 							Image:           c.image,
 							Ports:           containerPorts(),
 							ImagePullPolicy: pullPolicy,
@@ -205,6 +211,24 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet {
 		}...)
 	}
 
+	if c.forWindows {
+		daemonSet.Spec.Template.Spec.SecurityContext = nil
+		daemonSet.Spec.Template.Spec.Containers[0].SecurityContext = nil
+		daemonSet.Spec.Template.Spec.NodeSelector = map[string]string{
+			"kubernetes.io/os": "windows",
+		}
+		daemonSet.Spec.Template.Spec.OS = &corev1.PodOS{
+			Name: "windows",
+		}
+	} else {
+		daemonSet.Spec.Template.Spec.NodeSelector = map[string]string{
+			"kubernetes.io/os": "linux",
+		}
+		daemonSet.Spec.Template.Spec.OS = &corev1.PodOS{
+			Name: "linux",
+		}
+	}
+
 	daemonSet.Spec.Template.Spec.Containers[0].Env = append(daemonSet.Spec.Template.Spec.Containers[0].Env, c.envVars...)
 
 	return daemonSet

pkg/install/daemonset_test.go

@@ -24,10 +24,23 @@ import (
 )
 
 func TestDaemonSet(t *testing.T) {
+	userID := int64(0)
+	boolFalse := false
+	boolTrue := true
+
 	ds := DaemonSet("velero")
 
 	assert.Equal(t, "node-agent", ds.Spec.Template.Spec.Containers[0].Name)
 	assert.Equal(t, "velero", ds.ObjectMeta.Namespace)
+	assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["name"])
+	assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"])
+	assert.Equal(t, "linux", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"])
+	assert.Equal(t, "linux", string(ds.Spec.Template.Spec.OS.Name))
+	assert.Equal(t, corev1.PodSecurityContext{RunAsUser: &userID}, *ds.Spec.Template.Spec.SecurityContext)
+	assert.Equal(t, corev1.SecurityContext{Privileged: &boolFalse}, *ds.Spec.Template.Spec.Containers[0].SecurityContext)
+
+	ds = DaemonSet("velero", WithPrivilegedNodeAgent(true))
+	assert.Equal(t, corev1.SecurityContext{Privileged: &boolTrue}, *ds.Spec.Template.Spec.Containers[0].SecurityContext)
 
 	ds = DaemonSet("velero", WithImage("velero/velero:v0.11"))
 	assert.Equal(t, "velero/velero:v0.11", ds.Spec.Template.Spec.Containers[0].Image)
@@ -47,4 +60,14 @@ func TestDaemonSet(t *testing.T) {
 
 	ds = DaemonSet("velero", WithServiceAccountName("test-sa"))
 	assert.Equal(t, "test-sa", ds.Spec.Template.Spec.ServiceAccountName)
+
+	ds = DaemonSet("velero", WithForWindows())
+	assert.Equal(t, "node-agent-windows", ds.Spec.Template.Spec.Containers[0].Name)
+	assert.Equal(t, "velero", ds.ObjectMeta.Namespace)
+	assert.Equal(t, "node-agent-windows", ds.Spec.Template.ObjectMeta.Labels["name"])
+	assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"])
+	assert.Equal(t, "windows", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"])
+	assert.Equal(t, "windows", string(ds.Spec.Template.Spec.OS.Name))
+	assert.Equal(t, (*corev1.PodSecurityContext)(nil), ds.Spec.Template.Spec.SecurityContext)
+	assert.Equal(t, (*corev1.SecurityContext)(nil), ds.Spec.Template.Spec.Containers[0].SecurityContext)
 }

pkg/install/deployment.go

@@ -58,6 +58,7 @@ type podTemplateConfig struct {
 	repoMaintenanceJobConfigMap     string
 	nodeAgentConfigMap              string
 	itemBlockWorkerCount            int
+	forWindows                      bool
 }
 
 func WithImage(image string) podTemplateOption {
@@ -219,6 +220,12 @@ func WithItemBlockWorkerCount(itemBlockWorkerCount int) podTemplateOption {
 	}
 }
 
+func WithForWindows() podTemplateOption {
+	return func(c *podTemplateConfig) {
+		c.forWindows = true
+	}
+}
+
 func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment {
 	// TODO: Add support for server args
 	c := &podTemplateConfig{
@@ -324,6 +331,12 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment
 				Spec: corev1.PodSpec{
 					RestartPolicy:      corev1.RestartPolicyAlways,
 					ServiceAccountName: c.serviceAccountName,
+					NodeSelector: map[string]string{
+						"kubernetes.io/os": "linux",
+					},
+					OS: &corev1.PodOS{
+						Name: "linux",
+					},
 					Containers: []corev1.Container{
 						{
 							Name:            "velero",

pkg/install/deployment_test.go

@@ -99,4 +99,7 @@ func TestDeployment(t *testing.T) {
 	deploy = Deployment("velero", WithRepoMaintenanceJobConfigMap("test-repo-maintenance-config"))
 	assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2)
 	assert.Equal(t, "--repo-maintenance-job-configmap=test-repo-maintenance-config", deploy.Spec.Template.Spec.Containers[0].Args[1])
+
+	assert.Equal(t, "linux", deploy.Spec.Template.Spec.NodeSelector["kubernetes.io/os"])
+	assert.Equal(t, "linux", string(deploy.Spec.Template.Spec.OS.Name))
 }

pkg/install/install.go

@@ -206,9 +206,19 @@ func DeploymentIsReady(factory client.DynamicFactory, namespace string) (bool, e
 	return isReady, err
 }
 
-// DaemonSetIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that
+// NodeAgentIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that
 // pods are scheduled and available on all of the desired nodes.
-func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, error) {
+func NodeAgentIsReady(factory client.DynamicFactory, namespace string) (bool, error) {
+	return daemonSetIsReady(factory, namespace, "node-agent")
+}
+
+// NodeAgentWindowsIsReady will poll the Kubernetes API server to ensure the node-agent-windows daemonset is ready, i.e. that
+// pods are scheduled and available on all of the desired nodes.
+func NodeAgentWindowsIsReady(factory client.DynamicFactory, namespace string) (bool, error) {
+	return daemonSetIsReady(factory, namespace, "node-agent-windows")
+}
+
+func daemonSetIsReady(factory client.DynamicFactory, namespace string, name string) (bool, error) {
 	gvk := schema.FromAPIVersionAndKind(appsv1.SchemeGroupVersion.String(), "DaemonSet")
 	apiResource := metav1.APIResource{
 		Name:       "daemonsets",
@@ -225,7 +235,7 @@ func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, er
 	var readyObservations int32
 
 	err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) {
-		unstructuredDaemonSet, err := c.Get("node-agent", metav1.GetOptions{})
+		unstructuredDaemonSet, err := c.Get(name, metav1.GetOptions{})
 		if apierrors.IsNotFound(err) {
 			return false, nil
 		} else if err != nil {

pkg/install/install_test.go

@@ -127,7 +127,7 @@ func TestDeploymentIsReady(t *testing.T) {
 	assert.True(t, ready)
 }
 
-func TestDaemonSetIsReady(t *testing.T) {
+func TestNodeAgentIsReady(t *testing.T) {
 	daemonset := &appsv1.DaemonSet{
 		Status: appsv1.DaemonSetStatus{
 			NumberAvailable:        1,
@@ -143,7 +143,28 @@ func TestDaemonSetIsReady(t *testing.T) {
 	factory := &test.FakeDynamicFactory{}
 	factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil)
 
-	ready, err := DaemonSetIsReady(factory, "velero")
+	ready, err := NodeAgentIsReady(factory, "velero")
+	require.NoError(t, err)
+	assert.True(t, ready)
+}
+
+func TestNodeAgentWindowsIsReady(t *testing.T) {
+	daemonset := &appsv1.DaemonSet{
+		Status: appsv1.DaemonSetStatus{
+			NumberAvailable:        0,
+			DesiredNumberScheduled: 0,
+		},
+	}
+	obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(daemonset)
+	require.NoError(t, err)
+
+	dc := &test.FakeDynamicClient{}
+	dc.On("Get", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: obj}, nil)
+
+	factory := &test.FakeDynamicFactory{}
+	factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil)
+
+	ready, err := NodeAgentWindowsIsReady(factory, "velero")
 	require.NoError(t, err)
 	assert.True(t, ready)
 }

pkg/install/resources.go

@@ -246,6 +246,7 @@ type VeleroOptions struct {
 	SecretData                      []byte
 	RestoreOnly                     bool
 	UseNodeAgent                    bool
+	UseNodeAgentWindows             bool
 	PrivilegedNodeAgent             bool
 	UseVolumeSnapshots              bool
 	BSLConfig                       map[string]string
@@ -395,7 +396,7 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList {
 		fmt.Printf("error appending Deployment %s: %s\n", deploy.GetName(), err.Error())
 	}
 
-	if o.UseNodeAgent {
+	if o.UseNodeAgent || o.UseNodeAgentWindows {
 		dsOpts := []podTemplateOption{
 			WithAnnotations(o.PodAnnotations),
 			WithLabels(o.PodLabels),
@@ -414,9 +415,20 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList {
 			dsOpts = append(dsOpts, WithNodeAgentConfigMap(o.NodeAgentConfigMap))
 		}
 
-		ds := DaemonSet(o.Namespace, dsOpts...)
-		if err := appendUnstructured(resources, ds); err != nil {
-			fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error())
+		if o.UseNodeAgent {
+			ds := DaemonSet(o.Namespace, dsOpts...)
+			if err := appendUnstructured(resources, ds); err != nil {
+				fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error())
+			}
+		}
+
+		if o.UseNodeAgentWindows {
+			dsOpts = append(dsOpts, WithForWindows())
+
+			dsWin := DaemonSet(o.Namespace, dsOpts...)
+			if err := appendUnstructured(resources, dsWin); err != nil {
+				fmt.Printf("error appending DaemonSet %s: %s\n", dsWin.GetName(), err.Error())
+			}
 		}
 	}