From 19792652454cce504d79a320d6a29b1c6715bddb Mon Sep 17 00:00:00 2001 From: danfengl Date: Mon, 27 Nov 2023 01:53:13 +0000 Subject: [PATCH] Add sleep to avoid snapshot limitation issue Signed-off-by: danfengl --- test/e2e/backup/backup.go | 14 +++-- test/e2e/backups/deletion.go | 9 ++- test/e2e/backups/sync_backups.go | 4 +- test/e2e/backups/ttl.go | 5 +- .../api-group/enable_api_group_extentions.go | 2 +- .../api-group/enable_api_group_versions.go | 4 +- test/e2e/basic/namespace-mapping.go | 10 +-- test/e2e/bsl-mgmt/deletion.go | 5 +- test/e2e/e2e_suite_test.go | 10 +-- test/e2e/migration/migration.go | 9 +-- test/e2e/privilegesmgmt/ssr.go | 4 +- test/e2e/test/test.go | 12 ++-- test/e2e/upgrade/upgrade.go | 9 +-- test/perf/e2e_suite_test.go | 11 ++-- test/perf/test/test.go | 10 ++- test/types.go | 6 +- test/util/csi/common.go | 8 +-- test/util/kibishii/kibishii_utils.go | 14 ++--- test/util/providers/aws_utils.go | 61 ++++++++++++------- test/util/providers/common.go | 2 +- test/util/velero/install.go | 11 ++-- test/util/velero/velero_utils.go | 8 +-- 22 files changed, 126 insertions(+), 102 deletions(-) diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 923781e258..7fd31c7b97 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -77,7 +77,13 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } - + // [SKIP]: Static provisioning for vSphere CSI driver works differently from other drivers. + // For vSphere CSI, after you create a PV specifying an existing volume handle, CSI + // syncer will need to register it with CNS. For other CSI drivers, static provisioning + // usually does not go through storage system at all. That's probably why it took longer + if backupRestoreTestConfig.isRetainPVTest && veleroCfg.CloudProvider == "vsphere" { + Skip("Skip due to vSphere CSI driver long time issue of Static provisioning") + } var err error flag.Parse() UUIDgen, err = uuid.NewRandom() @@ -95,7 +101,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { } DeleteStorageClass(context.Background(), *veleroCfg.ClientToInstallVelero, KibishiiStorageClassName) }) - if veleroCfg.InstallVelero { + if InstallVelero { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() err = VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace) @@ -106,7 +112,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { When("kibishii is the sample workload", func() { It("should be successfully backed up and restored to the default BackupStorageLocation", func() { - if veleroCfg.InstallVelero { + if InstallVelero { if useVolumeSnapshots { //Install node agent also veleroCfg.UseNodeAgent = useVolumeSnapshots @@ -147,7 +153,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { if veleroCfg.AdditionalBSLCredentials == "" { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - if veleroCfg.InstallVelero { + if InstallVelero { if useVolumeSnapshots { veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots } else { //FS volume backup diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 34e502df02..76b078532b 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -59,8 +59,8 @@ func backup_deletion_test(useVolumeSnapshots bool) { } var err error flag.Parse() - if veleroCfg.InstallVelero { - Expect(PrepareVelero(context.Background(), "backup deletion")).To(Succeed()) + if InstallVelero { + Expect(PrepareVelero(context.Background(), "backup deletion", veleroCfg)).To(Succeed()) } UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) @@ -178,6 +178,11 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } + + // Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded + // Sleep for more than 15 seconds to avoid this issue. + time.Sleep(1 * time.Minute) + backupName = "backup-1-" + UUIDgen.String() BackupCfg.BackupName = backupName diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index 8f40c5786c..9a5c81113d 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -56,7 +56,7 @@ func BackupsSyncTest() { BeforeEach(func() { flag.Parse() - if VeleroCfg.InstallVelero { + if InstallVelero { veleroCfg := VeleroCfg veleroCfg.UseVolumeSnapshots = false Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) @@ -68,7 +68,7 @@ func BackupsSyncTest() { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) }) - if VeleroCfg.InstallVelero { + if InstallVelero { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 52069863cd..9c353eb346 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -32,7 +32,6 @@ import ( . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/kibishii" . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" @@ -66,7 +65,7 @@ func TTLTest() { BeforeEach(func() { flag.Parse() veleroCfg = VeleroCfg - if veleroCfg.InstallVelero { + if InstallVelero { // Make sure GCFrequency is shorter than backup TTL veleroCfg.GCFrequency = "4m0s" veleroCfg.UseVolumeSnapshots = useVolumeSnapshots @@ -82,7 +81,7 @@ func TTLTest() { }) ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - if veleroCfg.InstallVelero { + if InstallVelero { Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) } Expect(DeleteNamespace(ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) diff --git a/test/e2e/basic/api-group/enable_api_group_extentions.go b/test/e2e/basic/api-group/enable_api_group_extentions.go index 7e3466425b..5c77438fc3 100644 --- a/test/e2e/basic/api-group/enable_api_group_extentions.go +++ b/test/e2e/basic/api-group/enable_api_group_extentions.go @@ -71,7 +71,7 @@ func APIExtensionsVersionsTest() { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), *veleroCfg.DefaultClient) }) - if veleroCfg.InstallVelero { + if InstallVelero { By("Uninstall Velero and delete CRD ", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index 2d74d144e8..47a3c34205 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -69,7 +69,7 @@ func APIGropuVersionsTest() { flag.Parse() // TODO: install Velero once for the test suite once feature flag is // removed and velero installation becomes the same as other e2e tests. - if veleroCfg.InstallVelero { + if InstallVelero { veleroCfg.Features = "EnableAPIGroupVersions" veleroCfg.UseVolumeSnapshots = false err = VeleroInstall(context.Background(), &veleroCfg, false) @@ -96,7 +96,7 @@ func APIGropuVersionsTest() { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), *veleroCfg.ClientToInstallVelero) }) - if veleroCfg.InstallVelero { + if InstallVelero { By("Uninstall Velero in api group version case", func() { Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred()) }) diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index dbf98c1f92..27e74af656 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -64,7 +64,7 @@ func (n *NamespaceMapping) Init() error { n.MappedNamespaceList = mappedNSList fmt.Println(mappedNSList) n.BackupArgs = []string{ - "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName, + "create", "--namespace", n.VeleroCfg.VeleroNamespace, "backup", n.BackupName, "--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait", } if n.UseVolumeSnapshots { @@ -74,7 +74,7 @@ func (n *NamespaceMapping) Init() error { n.BackupArgs = append(n.BackupArgs, "--default-volumes-to-fs-backup") } n.RestoreArgs = []string{ - "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName, + "create", "--namespace", n.VeleroCfg.VeleroNamespace, "restore", n.RestoreName, "--from-backup", n.BackupName, "--namespace-mappings", mappedNS, "--wait", } @@ -89,9 +89,9 @@ func (n *NamespaceMapping) CreateResources() error { Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) }) By("Deploy sample workload of Kibishii", func() { - Expect(KibishiiPrepareBeforeBackup(n.Ctx, n.Client, VeleroCfg.CloudProvider, - ns, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, - VeleroCfg.KibishiiDirectory, false, n.kibishiiData)).To(Succeed()) + Expect(KibishiiPrepareBeforeBackup(n.Ctx, n.Client, n.VeleroCfg.CloudProvider, + ns, n.VeleroCfg.RegistryCredentialFile, n.VeleroCfg.Features, + n.VeleroCfg.KibishiiDirectory, false, n.kibishiiData)).To(Succeed()) }) } return nil diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 90b3abf570..909458e034 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -30,7 +30,6 @@ import ( . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/kibishii" - . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -69,8 +68,8 @@ func BslDeletionTest(useVolumeSnapshots bool) { flag.Parse() UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) - if veleroCfg.InstallVelero { - Expect(PrepareVelero(context.Background(), "BSL Deletion")).To(Succeed()) + if InstallVelero { + Expect(PrepareVelero(context.Background(), "BSL Deletion", veleroCfg)).To(Succeed()) } }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f14601924f..8d8ea6e295 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -50,7 +50,7 @@ import ( ) func init() { - VeleroCfg.Options = &install.Options{} + VeleroCfg.Options = install.Options{} flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.") flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") @@ -69,7 +69,7 @@ func init() { flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into") - flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") + flag.BoolVar(&InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") flag.BoolVar(&VeleroCfg.UseNodeAgent, "use-node-agent", true, "whether deploy node agent daemonset velero during the test. Optional.") flag.BoolVar(&VeleroCfg.UseVolumeSnapshots, "use-volume-snapshots", true, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.") flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") @@ -209,14 +209,14 @@ func TestE2e(t *testing.T) { } var _ = BeforeSuite(func() { - if VeleroCfg.InstallVelero { + if InstallVelero { By("Install test resources before testing") - Expect(PrepareVelero(context.Background(), "install resource before testing")).To(Succeed()) + Expect(PrepareVelero(context.Background(), "install resource before testing", VeleroCfg)).To(Succeed()) } }) var _ = AfterSuite(func() { - if VeleroCfg.InstallVelero && !VeleroCfg.Debug { + if InstallVelero && !VeleroCfg.Debug { By("release test resources after testing") ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index da808ba92c..2d5599c67a 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -63,15 +63,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } - if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" && !veleroCfg.SnapshotMoveData { - Skip("Volume snapshots migration not supported on AWS provisioned by Sheperd public pool") - } if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" { Skip("Migration test needs 2 clusters") } // need to uninstall Velero first in case of the affection of the existing global velero installation - if veleroCfg.InstallVelero { + if InstallVelero { By("Uninstall Velero", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() @@ -100,7 +97,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true) }) - if veleroCfg.InstallVelero { + if InstallVelero { By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() { DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true) }) @@ -258,7 +255,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) } if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && - strings.EqualFold(veleroCfg.Features, "EnableCSI") && + strings.EqualFold(veleroCfg.Features, FeatureCSI) && !OriginVeleroCfg.SnapshotMoveData { By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { time.Sleep(5 * time.Minute) diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go index 2ce784afdd..30ae1c11c5 100644 --- a/test/e2e/privilegesmgmt/ssr.go +++ b/test/e2e/privilegesmgmt/ssr.go @@ -43,8 +43,8 @@ func SSRTest() { BeforeEach(func() { flag.Parse() veleroCfg.UseVolumeSnapshots = false - if veleroCfg.InstallVelero { - Expect(PrepareVelero(context.Background(), "SSR test")).To(Succeed()) + if InstallVelero { + Expect(PrepareVelero(context.Background(), "SSR test", veleroCfg)).To(Succeed()) } }) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 995a4cf585..eaf9eda455 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -26,7 +26,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -83,15 +82,15 @@ func TestFunc(test VeleroBackupRestoreTest) func() { BeforeEach(func() { flag.Parse() // Using the global velero config which covered the installation for most common cases - veleroCfg := VeleroCfg + veleroCfg := test.GetTestCase().VeleroCfg // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed // TODO: Although this issue is closed, but it's not fixed. // TODO: After bump up k8s version in AWS pipeline, this issue also apply for AWS pipeline. if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") } - if veleroCfg.InstallVelero { - Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName)).To(Succeed()) + if InstallVelero { + Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, veleroCfg)).To(Succeed()) } }) It(test.GetTestMsg().Text, func() { @@ -102,7 +101,6 @@ func TestFunc(test VeleroBackupRestoreTest) func() { func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { return func() { - veleroCfg := VeleroCfg for k := range tests { Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) defer tests[k].GetTestCase().CtxCancel() @@ -110,8 +108,8 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { BeforeEach(func() { flag.Parse() - if veleroCfg.InstallVelero { - Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName)).To(Succeed()) + if InstallVelero { + Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName, tests[0].GetTestCase().VeleroCfg)).To(Succeed()) } }) diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index c9e9af90bb..af8e2dc93e 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -61,7 +61,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC BeforeEach(func() { veleroCfg = VeleroCfg - if !veleroCfg.InstallVelero { + veleroCfg.IsUpgradeTest = true + if !InstallVelero { Skip("Upgrade test should not be triggered if veleroCfg.InstallVelero is set to false") } if (len(veleroCfg.UpgradeFromVeleroVersion)) == 0 { @@ -74,7 +75,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC Skip("VeleroCLI should be provide") } // need to uninstall Velero first in case of the affection of the existing global velero installation - if veleroCfg.InstallVelero { + if InstallVelero { By("Uninstall Velero", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() @@ -91,7 +92,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC By(fmt.Sprintf("Delete sample workload namespace %s", upgradeNamespace), func() { DeleteNamespace(context.Background(), *veleroCfg.ClientToInstallVelero, upgradeNamespace, true) }) - if veleroCfg.InstallVelero { + if InstallVelero { By("Uninstall Velero", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() @@ -210,7 +211,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC fmt.Sprintf("failed to delete namespace %s", upgradeNamespace)) }) - if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") { + if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, FeatureCSI) { // Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before // v1.0 have issues to fail upgrade case. By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index 57599ec364..7d1cae16fd 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -30,7 +30,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" . "github.com/vmware-tanzu/velero/test" - "github.com/vmware-tanzu/velero/test/perf/backup" "github.com/vmware-tanzu/velero/test/perf/basic" "github.com/vmware-tanzu/velero/test/perf/restore" @@ -41,7 +40,7 @@ import ( ) func init() { - VeleroCfg.Options = &install.Options{} + VeleroCfg.Options = install.Options{} flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.") flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") @@ -56,7 +55,7 @@ func init() { flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into") - flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") + flag.BoolVar(&InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") flag.BoolVar(&VeleroCfg.UseNodeAgent, "use-node-agent", true, "whether deploy node agent daemonset velero during the test. Optional.") flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") flag.StringVar(&VeleroCfg.NodeAgentPodCPULimit, "node-agent-pod-cpu-limit", "4", "CPU limit for node agent pod. Optional.") @@ -120,15 +119,15 @@ func TestE2e(t *testing.T) { } var _ = BeforeSuite(func() { - if VeleroCfg.InstallVelero { + if InstallVelero { By("Install test resources before testing BeforeSuite") - Expect(PrepareVelero(context.Background(), "install resource before testing")).To(Succeed()) + Expect(PrepareVelero(context.Background(), "install resource before testing", VeleroCfg)).To(Succeed()) } }) var _ = AfterSuite(func() { Expect(report.GenerateYamlReport()).To(Succeed()) - if VeleroCfg.InstallVelero && !VeleroCfg.Debug { + if InstallVelero && !VeleroCfg.Debug { By("release test resources after testing") Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) } diff --git a/test/perf/test/test.go b/test/perf/test/test.go index c7f80e3fea..371b71047b 100644 --- a/test/perf/test/test.go +++ b/test/perf/test/test.go @@ -24,16 +24,14 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/pkg/errors" - "github.com/vmware-tanzu/velero/test/util/report" - "github.com/vmware-tanzu/velero/test/util/velero" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test" "github.com/vmware-tanzu/velero/test/perf/metrics" . "github.com/vmware-tanzu/velero/test/util/k8s" + "github.com/vmware-tanzu/velero/test/util/report" + "github.com/vmware-tanzu/velero/test/util/velero" . "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -86,8 +84,8 @@ func TestFunc(test VeleroBackupRestoreTest) func() { By(fmt.Sprintf("Run test %s ...... \n", test.GetTestCase().CaseBaseName)) BeforeEach(func() { // Using the global velero config which covered the installation for most common cases - if VeleroCfg.InstallVelero { - Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName)).To(Succeed()) + if InstallVelero { + Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, VeleroCfg)).To(Succeed()) } }) It(test.GetTestMsg().Text, func() { diff --git a/test/types.go b/test/types.go index 360c904735..ce6ecaa1b1 100644 --- a/test/types.go +++ b/test/types.go @@ -27,7 +27,9 @@ import ( const StorageClassName = "e2e-storage-class" const StorageClassName2 = "e2e-storage-class-2" +const FeatureCSI = "EnableCSI" +var InstallVelero bool var UUIDgen uuid.UUID var VeleroCfg VeleroConfig @@ -41,7 +43,7 @@ var ReportData *Report type VeleroConfig struct { VeleroCfgInPerf - *install.Options + install.Options VeleroCLI string VeleroImage string VeleroVersion string @@ -66,7 +68,6 @@ type VeleroConfig struct { MigrateFromVeleroCLI string Plugins string AddBSLPlugins string - InstallVelero bool KibishiiDirectory string Debug bool GCFrequency string @@ -83,6 +84,7 @@ type VeleroConfig struct { StandbyClusterPlugins string StandbyClusterOjbectStoreProvider string DebugVeleroPodRestart bool + IsUpgradeTest bool } type VeleroCfgInPerf struct { diff --git a/test/util/csi/common.go b/test/util/csi/common.go index 932646f0cf..76ba87c2c2 100644 --- a/test/util/csi/common.go +++ b/test/util/csi/common.go @@ -85,7 +85,7 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error } if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s", backupName) + fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) } return snapshotHandleList, nil } @@ -122,7 +122,7 @@ func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, err } if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s", backupName) + fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) } return snapshotHandleList, nil } @@ -179,8 +179,8 @@ func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount i return nil, errors.New("API version is invalid") } if len(snapshotContentNameList) != expectedCount { - return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) + return nil, errors.New(fmt.Sprintf("Snapshot content count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) } - fmt.Println(snapshotContentNameList) + fmt.Printf("snapshotContentNameList: %v \n", snapshotContentNameList) return snapshotContentNameList, nil } diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index de30dccf44..8a72050db5 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -151,7 +151,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } else { // wait for a period to confirm no snapshots exist for the backup time.Sleep(5 * time.Minute) - if strings.EqualFold(veleroFeatures, "EnableCSI") { + if strings.EqualFold(veleroFeatures, FeatureCSI) { _, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 0, kibishiiNamespace, backupName, KibishiiPVCNameList) if err != nil { @@ -214,10 +214,10 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if useVolumeSnapshots { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } + // if useVolumeSnapshots { + // fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") + // time.Sleep(5 * time.Minute) + // } fmt.Printf("VeleroRestore %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName, ""); err != nil { @@ -245,11 +245,11 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures, kibishiiDirectory string, useVolumeSnapshots bool, workerReplicas int) error { if strings.EqualFold(cloudPlatform, "azure") && - strings.EqualFold(veleroFeatures, "EnableCSI") { + strings.EqualFold(veleroFeatures, FeatureCSI) { cloudPlatform = "azure-csi" } if strings.EqualFold(cloudPlatform, "aws") && - strings.EqualFold(veleroFeatures, "EnableCSI") { + strings.EqualFold(veleroFeatures, FeatureCSI) { cloudPlatform = "aws-csi" } // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories diff --git a/test/util/providers/aws_utils.go b/test/util/providers/aws_utils.go index 9f70a6708e..31b3326d2d 100644 --- a/test/util/providers/aws_utils.go +++ b/test/util/providers/aws_utils.go @@ -25,16 +25,14 @@ import ( "os" "strings" - "github.com/aws/aws-sdk-go-v2/config" - s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/pkg/errors" "golang.org/x/net/context" @@ -310,19 +308,27 @@ func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj if region == "minio" { return errors.New("No snapshot for Minio provider") } + cfg, err := newAWSConfig(region, "", cloudCredentialsFile, false, "") if err != nil { return errors.Wrapf(err, "Failed to create AWS config of region %s", region) } + ec2Client := ec2.NewFromConfig(cfg) input := &ec2.DescribeSnapshotsInput{ OwnerIds: []string{"self"}, - Filters: []ec2types.Filter{ - { - Name: aws.String("tag:velero.io/backup"), - Values: []string{backupObject}, + } + + if !snapshotCheck.EnableCSI { + input = &ec2.DescribeSnapshotsInput{ + OwnerIds: []string{"self"}, + Filters: []ec2types.Filter{ + { + Name: aws.String("tag:velero.io/backup"), + Values: []string{backupObject}, + }, }, - }, + } } result, err := ec2Client.DescribeSnapshots(context.Background(), input) @@ -330,21 +336,34 @@ func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj fmt.Println(err) } - for _, n := range result.Snapshots { - fmt.Println(n.SnapshotId) - if n.SnapshotId != nil { - fmt.Println(*n.SnapshotId) + var actualCount int + if snapshotCheck.EnableCSI { + for _, snapshotId := range snapshotCheck.SnapshotIDList { + for _, n := range result.Snapshots { + if n.SnapshotId != nil && (*n.SnapshotId == snapshotId) { + actualCount++ + fmt.Printf("SnapshotId: %v, Tags: %v \n", *n.SnapshotId, n.Tags) + if n.VolumeId != nil { + fmt.Printf("VolumeId: %v \n", *n.VolumeId) + } + } + } } - fmt.Println(n.Tags) - fmt.Println(n.VolumeId) - if n.VolumeId != nil { - fmt.Println(*n.VolumeId) + } else { + for _, n := range result.Snapshots { + if n.SnapshotId != nil { + fmt.Printf("SnapshotId: %v, Tags: %v \n", *n.SnapshotId, n.Tags) + if n.VolumeId != nil { + fmt.Printf("VolumeId: %v \n", *n.VolumeId) + } + } } + actualCount = len(result.Snapshots) } - if len(result.Snapshots) != snapshotCheck.ExpectCount { - return errors.New(fmt.Sprintf("Snapshot count is not as expected %d", snapshotCheck.ExpectCount)) + if actualCount != snapshotCheck.ExpectCount { + return errors.New(fmt.Sprintf("Snapshot count %d is not as expected %d", actualCount, snapshotCheck.ExpectCount)) } else { - fmt.Printf("Snapshot count %d is as expected %d\n", len(result.Snapshots), snapshotCheck.ExpectCount) + fmt.Printf("Snapshot count %d is as expected %d\n", actualCount, snapshotCheck.ExpectCount) return nil } } diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 6502fbe201..0764e70de5 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -168,7 +168,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig } else { err = s.IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName, snapshotCheck) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup%s", backupName)) + return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup %s", backupName)) } } return nil diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 2c45b4c2dd..e2cbdb1ffe 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -57,6 +57,7 @@ type installOptions struct { func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluster bool) error { fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) + // veleroCfg struct including a set of BSL params and a set of additional BSL params, // additional BSL set is for additional BSL test only, so only default BSL set is effective // for VeleroInstall(). @@ -247,7 +248,7 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options fmt.Println("Start to install Azure VolumeSnapshotClass ...") if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if strings.EqualFold(options.Features, "EnableCSI") && options.UseVolumeSnapshots { + if strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { if strings.EqualFold(cloudProvider, "azure") { if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { return err @@ -578,13 +579,13 @@ func IsVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) (bo return true, nil } -func PrepareVelero(ctx context.Context, caseName string) error { - ready, err := IsVeleroReady(context.Background(), VeleroCfg.VeleroNamespace, VeleroCfg.UseNodeAgent) +func PrepareVelero(ctx context.Context, caseName string, veleroCfg VeleroConfig) error { + ready, err := IsVeleroReady(context.Background(), veleroCfg.VeleroNamespace, veleroCfg.UseNodeAgent) if err != nil { fmt.Printf("error in checking velero status with %v", err) ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace) ready = false } if ready { @@ -592,7 +593,7 @@ func PrepareVelero(ctx context.Context, caseName string) error { return nil } fmt.Printf("need to install velero for case %s \n", caseName) - return VeleroInstall(context.Background(), &VeleroCfg, false) + return VeleroInstall(context.Background(), &veleroCfg, false) } func VeleroUninstall(ctx context.Context, cli, namespace string) error { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index fd0d919e4f..d1a6b0c5b0 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -141,7 +141,7 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st plugins = append(plugins, pluginsForObjectStoreProvider...) } - if strings.EqualFold(feature, "EnableCSI") { + if strings.EqualFold(feature, FeatureCSI) { pluginsForFeature, ok = cloudMap["csi"] if !ok { return nil, errors.Errorf("fail to get CSI plugins by version: %s ", version) @@ -640,7 +640,7 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { return nil, errors.WithMessage(err, "failed to get velero version") } } - if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" { + if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" && !veleroCfg.IsUpgradeTest { needDataMoverPlugin = true } plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature, needDataMoverPlugin) @@ -1199,7 +1199,7 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCoun snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = KibishiiPVCNameList - if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + if (VeleroCfg.CloudProvider == "azure" || VeleroCfg.CloudProvider == "aws") && strings.EqualFold(VeleroCfg.Features, FeatureCSI) { snapshotCheckPoint.EnableCSI = true resourceName := "snapshot.storage.k8s.io" @@ -1215,7 +1215,7 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCoun return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } } - fmt.Println(snapshotCheckPoint) + fmt.Printf("snapshotCheckPoint: %v \n", snapshotCheckPoint) return snapshotCheckPoint, nil }