From 0ea4eb563adb286582bc2180fce4b6bf900b7b4e Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Tue, 26 Nov 2024 19:21:23 +0800 Subject: [PATCH 1/6] hybrid deploy Signed-off-by: Lyndon-Li --- pkg/install/daemonset.go | 32 ++++++++++++++++--- pkg/install/deployment.go | 7 ++++ pkg/install/resources.go | 7 ++++ pkg/plugin/clientmgmt/process/registry.go | 11 +++++++ .../clientmgmt/process/registry_test.go | 6 +++- 5 files changed, 58 insertions(+), 5 deletions(-) diff --git a/pkg/install/daemonset.go b/pkg/install/daemonset.go index 9cc3a814c1..81b06b64c3 100644 --- a/pkg/install/daemonset.go +++ b/pkg/install/daemonset.go @@ -57,8 +57,13 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet { userID := int64(0) mountPropagationMode := corev1.MountPropagationHostToContainer + dsName := "node-agent" + if c.forWindows { + dsName = "node-agent-windows" + } + daemonSet := &appsv1.DaemonSet{ - ObjectMeta: objectMeta(namespace, "node-agent"), + ObjectMeta: objectMeta(namespace, dsName), TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: appsv1.SchemeGroupVersion.String(), @@ -66,13 +71,14 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet { Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "name": "node-agent", + "name": dsName, }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: podLabels(c.labels, map[string]string{ - "name": "node-agent", + "name": dsName, + "role": "node-agent", }), Annotations: c.annotations, }, @@ -107,7 +113,7 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet { }, Containers: []corev1.Container{ { - Name: "node-agent", + Name: dsName, Image: c.image, Ports: containerPorts(), ImagePullPolicy: pullPolicy, @@ -205,6 +211,24 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet { }...) } + if c.forWindows { + daemonSet.Spec.Template.Spec.SecurityContext = nil + daemonSet.Spec.Template.Spec.Containers[0].SecurityContext = nil + daemonSet.Spec.Template.Spec.NodeSelector = map[string]string{ + "kubernetes.io/os": "windows", + } + daemonSet.Spec.Template.Spec.OS = &corev1.PodOS{ + Name: "windows", + } + } else { + daemonSet.Spec.Template.Spec.NodeSelector = map[string]string{ + "kubernetes.io/os": "linux", + } + daemonSet.Spec.Template.Spec.OS = &corev1.PodOS{ + Name: "linux", + } + } + daemonSet.Spec.Template.Spec.Containers[0].Env = append(daemonSet.Spec.Template.Spec.Containers[0].Env, c.envVars...) return daemonSet diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 0c9a0adf48..d074b8a7ea 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -58,6 +58,7 @@ type podTemplateConfig struct { repoMaintenanceJobConfigMap string nodeAgentConfigMap string itemBlockWorkerCount int + forWindows bool } func WithImage(image string) podTemplateOption { @@ -219,6 +220,12 @@ func WithItemBlockWorkerCount(itemBlockWorkerCount int) podTemplateOption { } } +func WithForWinows() podTemplateOption { + return func(c *podTemplateConfig) { + c.forWindows = true + } +} + func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment { // TODO: Add support for server args c := &podTemplateConfig{ diff --git a/pkg/install/resources.go b/pkg/install/resources.go index a2851c7af5..9788d04196 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -418,6 +418,13 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { if err := appendUnstructured(resources, ds); err != nil { fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error()) } + + dsOpts = append(dsOpts, WithForWinows()) + + dsWin := DaemonSet(o.Namespace, dsOpts...) + if err := appendUnstructured(resources, dsWin); err != nil { + fmt.Printf("error appending DaemonSet %s: %s\n", dsWin.GetName(), err.Error()) + } } return resources diff --git a/pkg/plugin/clientmgmt/process/registry.go b/pkg/plugin/clientmgmt/process/registry.go index 6238c45fb0..744048690a 100644 --- a/pkg/plugin/clientmgmt/process/registry.go +++ b/pkg/plugin/clientmgmt/process/registry.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -153,6 +154,7 @@ func (r *registry) readPluginsDir(dir string) ([]string, error) { } if !executable(file) { + r.logger.Warnf("Searching plugin skip file %s, not executable, mode %v, ext %s", file.Name(), file.Mode(), strings.ToLower(filepath.Ext(file.Name()))) continue } @@ -163,6 +165,15 @@ func (r *registry) readPluginsDir(dir string) ([]string, error) { // executable determines if a file is executable. func executable(info os.FileInfo) bool { + return executableLinux(info) || executableWindows(info) +} + +func executableWindows(info os.FileInfo) bool { + ext := strings.ToLower(filepath.Ext(info.Name())) + return (ext == ".exe") +} + +func executableLinux(info os.FileInfo) bool { /* When we AND the mode with 0111: diff --git a/pkg/plugin/clientmgmt/process/registry_test.go b/pkg/plugin/clientmgmt/process/registry_test.go index 1f74ffca0c..0bcef7c0b4 100644 --- a/pkg/plugin/clientmgmt/process/registry_test.go +++ b/pkg/plugin/clientmgmt/process/registry_test.go @@ -114,7 +114,9 @@ func TestReadPluginsDir(t *testing.T) { WithFileAndMode("/plugins/nonexecutable2", []byte("plugin2"), 0644). WithFileAndMode("/plugins/executable3", []byte("plugin3"), 0755). WithFileAndMode("/plugins/nested/executable4", []byte("plugin4"), 0755). - WithFileAndMode("/plugins/nested/nonexecutable5", []byte("plugin4"), 0644) + WithFileAndMode("/plugins/nested/nonexecutable5", []byte("plugin4"), 0644). + WithFileAndMode("/plugins/nested/win-exe1.exe", []byte("plugin4"), 0600). + WithFileAndMode("/plugins/nested/WIN-EXE2.EXE", []byte("plugin4"), 0600) plugins, err := r.readPluginsDir(dir) require.NoError(t, err) @@ -123,6 +125,8 @@ func TestReadPluginsDir(t *testing.T) { "/plugins/executable1", "/plugins/executable3", "/plugins/nested/executable4", + "/plugins/nested/win-exe1.exe", + "/plugins/nested/WIN-EXE2.EXE", } sort.Strings(plugins) From 11cd6d922bb59429de11f51fcbf72aa9c855ca2d Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Wed, 4 Dec 2024 15:49:44 +0800 Subject: [PATCH 2/6] hybrid deploy Signed-off-by: Lyndon-Li --- changelogs/unreleased/8504-Lyndon-Li | 1 + pkg/cmd/cli/install/install.go | 7 +++- pkg/cmd/cli/nodeagent/server.go | 28 +++++++++++++--- pkg/cmd/server/server.go | 33 ++++++++++++++----- pkg/install/daemonset_test.go | 23 +++++++++++++ pkg/install/deployment.go | 2 +- pkg/install/install.go | 16 +++++++-- pkg/install/install_test.go | 25 ++++++++++++-- pkg/install/resources.go | 2 +- .../clientmgmt/process/registry_test.go | 24 ++++++++++++++ 10 files changed, 140 insertions(+), 21 deletions(-) create mode 100644 changelogs/unreleased/8504-Lyndon-Li diff --git a/changelogs/unreleased/8504-Lyndon-Li b/changelogs/unreleased/8504-Lyndon-Li new file mode 100644 index 0000000000..f6e877d954 --- /dev/null +++ b/changelogs/unreleased/8504-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8416, #8417, deploy Velero server and node-agent in linux/Windows hybrid env \ No newline at end of file diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 45be13d9fc..a72ac59724 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -394,7 +394,12 @@ func (o *Options) Run(c *cobra.Command, f client.Factory) error { if o.UseNodeAgent { fmt.Println("Waiting for node-agent daemonset to be ready.") - if _, err = install.DaemonSetIsReady(dynamicFactory, o.Namespace); err != nil { + if _, err = install.NodeAgentIsReady(dynamicFactory, o.Namespace); err != nil { + return errors.Wrap(err, errorMsg) + } + + fmt.Println("Waiting for node-agent-windows daemonset to be ready.") + if _, err = install.NodeAgentWindowsIsReady(dynamicFactory, o.Namespace); err != nil { return errors.Wrap(err, errorMsg) } } diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index ba7c1610fb..713d863593 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -200,13 +200,31 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi }, }, } - mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{ - Scheme: scheme, - Cache: cacheOption, - }) + + var mgr manager.Manager + retry := 10 + for { + mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{ + Scheme: scheme, + Cache: cacheOption, + }) + if err == nil { + break + } + + retry-- + if retry == 0 { + break + } + + logger.WithError(err).Warn("Failed to create controller manager, need retry") + + time.Sleep(time.Second) + } + if err != nil { cancelFunc() - return nil, err + return nil, errors.Wrap(err, "error creating controller manager") } s := &nodeAgentServer{ diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index b5f9576f88..a0828e69ef 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -239,17 +239,34 @@ func newServer(f client.Factory, config *config.Config, logger *logrus.Logger) ( ctrl.SetLogger(logrusr.New(logger)) - mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{ - Scheme: scheme, - Cache: cache.Options{ - DefaultNamespaces: map[string]cache.Config{ - f.Namespace(): {}, + var mgr manager.Manager + retry := 10 + for { + mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + f.Namespace(): {}, + }, }, - }, - }) + }) + if err == nil { + break + } + + retry-- + if retry == 0 { + break + } + + logger.WithError(err).Warn("Failed to create controller manager, need retry") + + time.Sleep(time.Second) + } + if err != nil { cancelFunc() - return nil, err + return nil, errors.Wrap(err, "error creating controller manager") } credentialFileStore, err := credentials.NewNamespacedFileStore( diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index c181306e2c..c0b289f091 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -24,10 +24,23 @@ import ( ) func TestDaemonSet(t *testing.T) { + userID := int64(0) + boolFalse := false + boolTrue := true + ds := DaemonSet("velero") assert.Equal(t, "node-agent", ds.Spec.Template.Spec.Containers[0].Name) assert.Equal(t, "velero", ds.ObjectMeta.Namespace) + assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["name"]) + assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) + assert.Equal(t, "linux", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) + assert.Equal(t, "linux", string(ds.Spec.Template.Spec.OS.Name)) + assert.Equal(t, corev1.PodSecurityContext{RunAsUser: &userID}, *ds.Spec.Template.Spec.SecurityContext) + assert.Equal(t, corev1.SecurityContext{Privileged: &boolFalse}, *ds.Spec.Template.Spec.Containers[0].SecurityContext) + + ds = DaemonSet("velero", WithPrivilegedNodeAgent(true)) + assert.Equal(t, corev1.SecurityContext{Privileged: &boolTrue}, *ds.Spec.Template.Spec.Containers[0].SecurityContext) ds = DaemonSet("velero", WithImage("velero/velero:v0.11")) assert.Equal(t, "velero/velero:v0.11", ds.Spec.Template.Spec.Containers[0].Image) @@ -47,4 +60,14 @@ func TestDaemonSet(t *testing.T) { ds = DaemonSet("velero", WithServiceAccountName("test-sa")) assert.Equal(t, "test-sa", ds.Spec.Template.Spec.ServiceAccountName) + + ds = DaemonSet("velero", WithForWindows()) + assert.Equal(t, "node-agent-windows", ds.Spec.Template.Spec.Containers[0].Name) + assert.Equal(t, "velero", ds.ObjectMeta.Namespace) + assert.Equal(t, "node-agent-windows", ds.Spec.Template.ObjectMeta.Labels["name"]) + assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) + assert.Equal(t, "windows", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) + assert.Equal(t, "windows", string(ds.Spec.Template.Spec.OS.Name)) + assert.Equal(t, (*corev1.PodSecurityContext)(nil), ds.Spec.Template.Spec.SecurityContext) + assert.Equal(t, (*corev1.SecurityContext)(nil), ds.Spec.Template.Spec.Containers[0].SecurityContext) } diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index d074b8a7ea..4ee8e52383 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -220,7 +220,7 @@ func WithItemBlockWorkerCount(itemBlockWorkerCount int) podTemplateOption { } } -func WithForWinows() podTemplateOption { +func WithForWindows() podTemplateOption { return func(c *podTemplateConfig) { c.forWindows = true } diff --git a/pkg/install/install.go b/pkg/install/install.go index 3bf0651702..391b97ec1b 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -206,9 +206,19 @@ func DeploymentIsReady(factory client.DynamicFactory, namespace string) (bool, e return isReady, err } -// DaemonSetIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that +// NodeAgentIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that // pods are scheduled and available on all of the desired nodes. -func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, error) { +func NodeAgentIsReady(factory client.DynamicFactory, namespace string) (bool, error) { + return daemonSetIsReady(factory, namespace, "node-agent") +} + +// NodeAgentWindowsIsReady will poll the Kubernetes API server to ensure the node-agent-windows daemonset is ready, i.e. that +// pods are scheduled and available on all of the desired nodes. +func NodeAgentWindowsIsReady(factory client.DynamicFactory, namespace string) (bool, error) { + return daemonSetIsReady(factory, namespace, "node-agent-windows") +} + +func daemonSetIsReady(factory client.DynamicFactory, namespace string, name string) (bool, error) { gvk := schema.FromAPIVersionAndKind(appsv1.SchemeGroupVersion.String(), "DaemonSet") apiResource := metav1.APIResource{ Name: "daemonsets", @@ -225,7 +235,7 @@ func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, er var readyObservations int32 err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - unstructuredDaemonSet, err := c.Get("node-agent", metav1.GetOptions{}) + unstructuredDaemonSet, err := c.Get(name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return false, nil } else if err != nil { diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go index 52bfbaf418..e9c39adf18 100644 --- a/pkg/install/install_test.go +++ b/pkg/install/install_test.go @@ -127,7 +127,7 @@ func TestDeploymentIsReady(t *testing.T) { assert.True(t, ready) } -func TestDaemonSetIsReady(t *testing.T) { +func TestNodeAgentIsReady(t *testing.T) { daemonset := &appsv1.DaemonSet{ Status: appsv1.DaemonSetStatus{ NumberAvailable: 1, @@ -143,7 +143,28 @@ func TestDaemonSetIsReady(t *testing.T) { factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) - ready, err := DaemonSetIsReady(factory, "velero") + ready, err := NodeAgentIsReady(factory, "velero") + require.NoError(t, err) + assert.True(t, ready) +} + +func TestNodeAgentWindowsIsReady(t *testing.T) { + daemonset := &appsv1.DaemonSet{ + Status: appsv1.DaemonSetStatus{ + NumberAvailable: 0, + DesiredNumberScheduled: 0, + }, + } + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(daemonset) + require.NoError(t, err) + + dc := &test.FakeDynamicClient{} + dc.On("Get", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: obj}, nil) + + factory := &test.FakeDynamicFactory{} + factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) + + ready, err := NodeAgentWindowsIsReady(factory, "velero") require.NoError(t, err) assert.True(t, ready) } diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 9788d04196..824e3e92c9 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -419,7 +419,7 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error()) } - dsOpts = append(dsOpts, WithForWinows()) + dsOpts = append(dsOpts, WithForWindows()) dsWin := DaemonSet(o.Namespace, dsOpts...) if err := appendUnstructured(resources, dsWin); err != nil { diff --git a/pkg/plugin/clientmgmt/process/registry_test.go b/pkg/plugin/clientmgmt/process/registry_test.go index 0bcef7c0b4..e188419b2f 100644 --- a/pkg/plugin/clientmgmt/process/registry_test.go +++ b/pkg/plugin/clientmgmt/process/registry_test.go @@ -45,6 +45,7 @@ func TestNewRegistry(t *testing.T) { type fakeFileInfo struct { fs.FileInfo + name string mode os.FileMode } @@ -52,9 +53,14 @@ func (f *fakeFileInfo) Mode() os.FileMode { return f.mode } +func (f *fakeFileInfo) Name() string { + return f.name +} + func TestExecutable(t *testing.T) { tests := []struct { name string + fileName string mode uint32 expectExecutable bool }{ @@ -90,11 +96,29 @@ func TestExecutable(t *testing.T) { mode: 0777, expectExecutable: true, }, + { + name: "windows lower case", + fileName: "test.exe", + mode: 0, + expectExecutable: true, + }, + { + name: "windows upper case", + fileName: "test.EXE", + mode: 0, + expectExecutable: true, + }, + { + name: "windows wrong ext", + fileName: "test.EXE1", + mode: 0, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { info := &fakeFileInfo{ + name: test.fileName, mode: os.FileMode(test.mode), } From a5a6e47e4253e2d34b8112fa403829420b740099 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Tue, 17 Dec 2024 13:27:51 +0800 Subject: [PATCH 3/6] add use-node-agent-windows Signed-off-by: Lyndon-Li --- pkg/cmd/cli/install/install.go | 5 ++++- pkg/install/resources.go | 21 +++++++++++++-------- pkg/install/resources_test.go | 19 +++++++++++-------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index a72ac59724..320de41596 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -66,6 +66,7 @@ type Options struct { BackupStorageConfig flag.Map VolumeSnapshotConfig flag.Map UseNodeAgent bool + UseNodeAgentWindows bool PrivilegedNodeAgent bool //TODO remove UseRestic when migration test out of using it UseRestic bool @@ -119,7 +120,8 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.UseVolumeSnapshots, "use-volume-snapshots", o.UseVolumeSnapshots, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.") flags.BoolVar(&o.RestoreOnly, "restore-only", o.RestoreOnly, "Run the server in restore-only mode. Optional.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Generate resources, but don't send them to the cluster. Use with -o. Optional.") - flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts Velero modules that need to run in one or more nodes(i.e. Restic, Kopia).") + flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts Velero modules that need to run in one or more Linux nodes(i.e. Restic, Kopia).") + flags.BoolVar(&o.UseNodeAgentWindows, "use-node-agent-windows", o.UseNodeAgentWindows, "Create Velero node-agent-windows daemonset. Optional. Velero node-agent-windows hosts Velero modules that need to run in one or more Windows nodes(i.e. Restic, Kopia).") flags.BoolVar(&o.PrivilegedNodeAgent, "privileged-node-agent", o.PrivilegedNodeAgent, "Use privileged mode for the node agent. Optional. Required to backup block devices.") flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.") flags.DurationVar(&o.DefaultRepoMaintenanceFrequency, "default-repo-maintain-frequency", o.DefaultRepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default. Optional.") @@ -269,6 +271,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { SecretData: secretData, RestoreOnly: o.RestoreOnly, UseNodeAgent: o.UseNodeAgent, + UseNodeAgentWindows: o.UseNodeAgentWindows, PrivilegedNodeAgent: o.PrivilegedNodeAgent, UseVolumeSnapshots: o.UseVolumeSnapshots, BSLConfig: o.BackupStorageConfig.Data(), diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 824e3e92c9..6560d2e153 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -246,6 +246,7 @@ type VeleroOptions struct { SecretData []byte RestoreOnly bool UseNodeAgent bool + UseNodeAgentWindows bool PrivilegedNodeAgent bool UseVolumeSnapshots bool BSLConfig map[string]string @@ -395,7 +396,7 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { fmt.Printf("error appending Deployment %s: %s\n", deploy.GetName(), err.Error()) } - if o.UseNodeAgent { + if o.UseNodeAgent || o.UseNodeAgentWindows { dsOpts := []podTemplateOption{ WithAnnotations(o.PodAnnotations), WithLabels(o.PodLabels), @@ -414,16 +415,20 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { dsOpts = append(dsOpts, WithNodeAgentConfigMap(o.NodeAgentConfigMap)) } - ds := DaemonSet(o.Namespace, dsOpts...) - if err := appendUnstructured(resources, ds); err != nil { - fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error()) + if o.UseNodeAgent { + ds := DaemonSet(o.Namespace, dsOpts...) + if err := appendUnstructured(resources, ds); err != nil { + fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error()) + } } - dsOpts = append(dsOpts, WithForWindows()) + if o.UseNodeAgentWindows { + dsOpts = append(dsOpts, WithForWindows()) - dsWin := DaemonSet(o.Namespace, dsOpts...) - if err := appendUnstructured(resources, dsWin); err != nil { - fmt.Printf("error appending DaemonSet %s: %s\n", dsWin.GetName(), err.Error()) + dsWin := DaemonSet(o.Namespace, dsOpts...) + if err := appendUnstructured(resources, dsWin); err != nil { + fmt.Printf("error appending DaemonSet %s: %s\n", dsWin.GetName(), err.Error()) + } } } diff --git a/pkg/install/resources_test.go b/pkg/install/resources_test.go index 58bbac58ca..5708e531f8 100644 --- a/pkg/install/resources_test.go +++ b/pkg/install/resources_test.go @@ -77,21 +77,22 @@ func TestAllCRDs(t *testing.T) { func TestAllResources(t *testing.T) { option := &VeleroOptions{ - Namespace: "velero", - SecretData: []byte{'a'}, - UseVolumeSnapshots: true, - UseNodeAgent: true, + Namespace: "velero", + SecretData: []byte{'a'}, + UseVolumeSnapshots: true, + UseNodeAgent: true, + UseNodeAgentWindows: true, } list := AllResources(option) - objects := map[string]unstructured.Unstructured{} + objects := map[string][]unstructured.Unstructured{} for _, item := range list.Items { - objects[item.GetKind()] = item + objects[item.GetKind()] = append(objects[item.GetKind()], item) } ns, exist := objects["Namespace"] require.True(t, exist) - assert.Equal(t, "velero", ns.GetName()) + assert.Equal(t, "velero", ns[0].GetName()) _, exist = objects["ClusterRoleBinding"] assert.True(t, exist) @@ -111,6 +112,8 @@ func TestAllResources(t *testing.T) { _, exist = objects["Deployment"] assert.True(t, exist) - _, exist = objects["DaemonSet"] + ds, exist := objects["DaemonSet"] assert.True(t, exist) + + assert.Equal(t, 2, len(ds)) } From fe0a45eac69328d3440869b188940884eea8543b Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Tue, 17 Dec 2024 13:30:45 +0800 Subject: [PATCH 4/6] restict velero server in linux nodes Signed-off-by: Lyndon-Li --- pkg/install/deployment.go | 6 ++++++ pkg/install/deployment_test.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 4ee8e52383..41fe474d0c 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -331,6 +331,12 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyAlways, ServiceAccountName: c.serviceAccountName, + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + OS: &corev1.PodOS{ + Name: "linux", + }, Containers: []corev1.Container{ { Name: "velero", diff --git a/pkg/install/deployment_test.go b/pkg/install/deployment_test.go index 48e02fb5e1..4f91563d18 100644 --- a/pkg/install/deployment_test.go +++ b/pkg/install/deployment_test.go @@ -99,4 +99,7 @@ func TestDeployment(t *testing.T) { deploy = Deployment("velero", WithRepoMaintenanceJobConfigMap("test-repo-maintenance-config")) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--repo-maintenance-job-configmap=test-repo-maintenance-config", deploy.Spec.Template.Spec.Containers[0].Args[1]) + + assert.Equal(t, "linux", deploy.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) + assert.Equal(t, "linux", string(deploy.Spec.Template.Spec.OS.Name)) } From 99ba81e5d1c2b920c4b37c3f13b13b361a779c92 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Tue, 17 Dec 2024 13:54:03 +0800 Subject: [PATCH 5/6] add use-node-agent-windows Signed-off-by: Lyndon-Li --- pkg/cmd/cli/install/install.go | 4 ++-- pkg/install/resources_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 0b3e2b21c9..e725c913dd 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -118,8 +118,8 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.UseVolumeSnapshots, "use-volume-snapshots", o.UseVolumeSnapshots, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.") flags.BoolVar(&o.RestoreOnly, "restore-only", o.RestoreOnly, "Run the server in restore-only mode. Optional.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Generate resources, but don't send them to the cluster. Use with -o. Optional.") - flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts Velero modules that need to run in one or more Linux nodes(i.e. Restic, Kopia).") - flags.BoolVar(&o.UseNodeAgentWindows, "use-node-agent-windows", o.UseNodeAgentWindows, "Create Velero node-agent-windows daemonset. Optional. Velero node-agent-windows hosts Velero modules that need to run in one or more Windows nodes(i.e. Restic, Kopia).") + flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts and associates Velero modules that need to run in one or more Linux nodes.") + flags.BoolVar(&o.UseNodeAgentWindows, "use-node-agent-windows", o.UseNodeAgentWindows, "Create Velero node-agent-windows daemonset. Optional. Velero node-agent-windows hosts and associates Velero modules that need to run in one or more Windows nodes.") flags.BoolVar(&o.PrivilegedNodeAgent, "privileged-node-agent", o.PrivilegedNodeAgent, "Use privileged mode for the node agent. Optional. Required to backup block devices.") flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.") flags.DurationVar(&o.DefaultRepoMaintenanceFrequency, "default-repo-maintain-frequency", o.DefaultRepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default. Optional.") diff --git a/pkg/install/resources_test.go b/pkg/install/resources_test.go index 5708e531f8..cb37b1e3fc 100644 --- a/pkg/install/resources_test.go +++ b/pkg/install/resources_test.go @@ -115,5 +115,5 @@ func TestAllResources(t *testing.T) { ds, exist := objects["DaemonSet"] assert.True(t, exist) - assert.Equal(t, 2, len(ds)) + assert.Len(t, ds, 2) } From 4ad9c2485ad2150b79a2ed590fe7297ab6f5c38f Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Wed, 18 Dec 2024 10:32:52 +0800 Subject: [PATCH 6/6] hybrid deploy Signed-off-by: Lyndon-Li --- pkg/exposer/csi_snapshot_test.go | 2 +- pkg/exposer/generic_restore_test.go | 2 +- pkg/nodeagent/node_agent.go | 7 +++++-- pkg/nodeagent/node_agent_test.go | 8 ++++---- pkg/podvolume/backupper_test.go | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index 77d7926356..9f2865f07c 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -1146,7 +1146,7 @@ func Test_csiSnapshotExposer_DiagnoseExpose(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "node-agent-pod-1", - Labels: map[string]string{"name": "node-agent"}, + Labels: map[string]string{"role": "node-agent"}, }, Spec: corev1.PodSpec{ NodeName: "fake-node", diff --git a/pkg/exposer/generic_restore_test.go b/pkg/exposer/generic_restore_test.go index 2eec0ce182..d2d56ece73 100644 --- a/pkg/exposer/generic_restore_test.go +++ b/pkg/exposer/generic_restore_test.go @@ -627,7 +627,7 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "node-agent-pod-1", - Labels: map[string]string{"name": "node-agent"}, + Labels: map[string]string{"role": "node-agent"}, }, Spec: corev1.PodSpec{ NodeName: "fake-node", diff --git a/pkg/nodeagent/node_agent.go b/pkg/nodeagent/node_agent.go index a57379f37f..898ea1e018 100644 --- a/pkg/nodeagent/node_agent.go +++ b/pkg/nodeagent/node_agent.go @@ -35,6 +35,9 @@ import ( const ( // daemonSet is the name of the Velero node agent daemonset. daemonSet = "node-agent" + + // nodeAgentRole marks pods with node-agent role on all nodes. + nodeAgentRole = "node-agent" ) var ( @@ -116,7 +119,7 @@ func isRunningInNode(ctx context.Context, namespace string, nodeName string, crC } pods := new(v1.PodList) - parsedSelector, err := labels.Parse(fmt.Sprintf("name=%s", daemonSet)) + parsedSelector, err := labels.Parse(fmt.Sprintf("role=%s", nodeAgentRole)) if err != nil { return errors.Wrap(err, "fail to parse selector") } @@ -128,7 +131,7 @@ func isRunningInNode(ctx context.Context, namespace string, nodeName string, crC } if err != nil { - return errors.Wrap(err, "failed to list daemonset pods") + return errors.Wrap(err, "failed to list node-agent pods") } for i := range pods.Items { diff --git a/pkg/nodeagent/node_agent_test.go b/pkg/nodeagent/node_agent_test.go index 1c24427b1f..700acdec8f 100644 --- a/pkg/nodeagent/node_agent_test.go +++ b/pkg/nodeagent/node_agent_test.go @@ -108,11 +108,11 @@ func TestIsRunningInNode(t *testing.T) { corev1.AddToScheme(scheme) nonNodeAgentPod := builder.ForPod("fake-ns", "fake-pod").Result() - nodeAgentPodNotRunning := builder.ForPod("fake-ns", "fake-pod").Labels(map[string]string{"name": "node-agent"}).Result() - nodeAgentPodRunning1 := builder.ForPod("fake-ns", "fake-pod-1").Labels(map[string]string{"name": "node-agent"}).Phase(corev1.PodRunning).Result() - nodeAgentPodRunning2 := builder.ForPod("fake-ns", "fake-pod-2").Labels(map[string]string{"name": "node-agent"}).Phase(corev1.PodRunning).Result() + nodeAgentPodNotRunning := builder.ForPod("fake-ns", "fake-pod").Labels(map[string]string{"role": "node-agent"}).Result() + nodeAgentPodRunning1 := builder.ForPod("fake-ns", "fake-pod-1").Labels(map[string]string{"role": "node-agent"}).Phase(corev1.PodRunning).Result() + nodeAgentPodRunning2 := builder.ForPod("fake-ns", "fake-pod-2").Labels(map[string]string{"role": "node-agent"}).Phase(corev1.PodRunning).Result() nodeAgentPodRunning3 := builder.ForPod("fake-ns", "fake-pod-3"). - Labels(map[string]string{"name": "node-agent"}). + Labels(map[string]string{"role": "node-agent"}). Phase(corev1.PodRunning). NodeName("fake-node"). Result() diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index fe50f9e30e..c21ad5ebb9 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -260,7 +260,7 @@ func createPodObj(running bool, withVolume bool, withVolumeMounted bool, volumeN func createNodeAgentPodObj(running bool) *corev1api.Pod { podObj := builder.ForPod(velerov1api.DefaultNamespace, "fake-node-agent").Result() - podObj.Labels = map[string]string{"name": "node-agent"} + podObj.Labels = map[string]string{"role": "node-agent"} if running { podObj.Status.Phase = corev1api.PodRunning