Add E2E test for taking CSI snapshot to PV with retain reclaim policy

Signed-off-by: danfengl <danfengl@vmware.com>

test/e2e/backup/backup.go

@@ -31,15 +31,33 @@ import (
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
 
+type BackupRestoreTestConfig struct {
+	useVolumeSnapshots  bool
+	kibishiiPatchSubDir string
+	isRetainPVTest      bool
+}
+
 func BackupRestoreWithSnapshots() {
-	BackupRestoreTest(true)
+	config := BackupRestoreTestConfig{true, "", false}
+	BackupRestoreTest(config)
 }
 
 func BackupRestoreWithRestic() {
-	BackupRestoreTest(false)
+	config := BackupRestoreTestConfig{false, "", false}
+	BackupRestoreTest(config)
+}
+
+func BackupRestoreRetainedPVWithSnapshots() {
+	config := BackupRestoreTestConfig{true, "overlays/sc-reclaim-policy/", true}
+	BackupRestoreTest(config)
+}
+
+func BackupRestoreRetainedPVWithRestic() {
+	config := BackupRestoreTestConfig{false, "overlays/sc-reclaim-policy/", true}
+	BackupRestoreTest(config)
 }
 
-func BackupRestoreTest(useVolumeSnapshots bool) {
+func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 
 	var (
 		backupName, restoreName, kibishiiNamespace string
@@ -48,25 +66,34 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 		veleroCfg                                  VeleroConfig
 	)
 	provideSnapshotVolumesParmInBackup = false
+	useVolumeSnapshots := backupRestoreTestConfig.useVolumeSnapshots
 
 	BeforeEach(func() {
 		veleroCfg = VeleroCfg
+
+		veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir
 		veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 		veleroCfg.UseNodeAgent = !useVolumeSnapshots
 		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
 			Skip("Volume snapshots not supported on kind")
 		}
+
 		var err error
 		flag.Parse()
 		UUIDgen, err = uuid.NewRandom()
 		kibishiiNamespace = "k-" + UUIDgen.String()
 		Expect(err).To(Succeed())
+		DeleteStorageClass(context.Background(), *veleroCfg.ClientToInstallVelero, KibishiiStorageClassName)
 	})
 
 	AfterEach(func() {
 		if !veleroCfg.Debug {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
+				if backupRestoreTestConfig.isRetainPVTest {
+					CleanAllRetainedPV(context.Background(), *veleroCfg.ClientToInstallVelero)
+				}
+				DeleteStorageClass(context.Background(), *veleroCfg.ClientToInstallVelero, KibishiiStorageClassName)
 			})
 			if veleroCfg.InstallVelero {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
@@ -106,6 +133,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 		})
 
 		It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
+			if backupRestoreTestConfig.isRetainPVTest {
+				Skip("It's tested by 1st test case")
+			}
 			if veleroCfg.AdditionalBSLProvider == "" {
 				Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
 			}

test/e2e/basic/namespace-mapping.go

@@ -102,7 +102,7 @@ func (n *NamespaceMapping) Verify() error {
 		n.kibishiiData.Levels = len(*n.NSIncluded) + index
 		By(fmt.Sprintf("Verify workload %s after restore ", ns), func() {
 			Expect(KibishiiVerifyAfterRestore(n.Client, ns,
-				n.Ctx, n.kibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+				n.Ctx, n.kibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 		})
 	}
 	for _, ns := range *n.NSIncluded {

test/e2e/e2e_suite_test.go

@@ -102,6 +102,10 @@ var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provi
 
 var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
 
+var _ = Describe("[Basic][Snapshot][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithSnapshots)
+
+var _ = Describe("[Basic][Restic][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithRestic)
+
 var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest)
 
 var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSBackupRestore)

test/e2e/migration/migration.go

@@ -275,13 +275,16 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
 				ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
 				Expect(ns.Name).To(Equal(migrationNamespace))
-				Expect(err).NotTo(HaveOccurred())
+				if err != nil {
+					fmt.Printf("debug-err: %v", err)
+				}
+				//Expect(err).NotTo(HaveOccurred(), err)
 
+				//Ensure cluster-B is the target cluster
 				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
 				_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
 				Expect(err).To(HaveOccurred())
 				strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")
-
 				fmt.Println(err)
 
 				veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
@@ -335,7 +338,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 
 			By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() {
 				Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace,
-					oneHourTimeout, &KibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+					oneHourTimeout, &KibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 			})
 
 			// TODO: delete backup created by case self, not all

test/e2e/pv-backup/pv-backup-filter.go

@@ -180,7 +180,7 @@ func fileContent(namespace, podName, volume string) string {
 }
 
 func fileExist(ctx context.Context, namespace, podName, volume string) error {
-	c, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
+	c, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ",
 			FILE_NAME, volume, podName, namespace))
@@ -195,7 +195,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error {
 	}
 }
 func fileNotExist(ctx context.Context, namespace, podName, volume string) error {
-	_, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
+	_, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return nil
 	} else {

test/e2e/resourcepolicies/resource_policies.go

@@ -24,7 +24,6 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -164,7 +163,7 @@ func (r *ResourcePoliciesCase) Verify() error {
 					if vol.Name != volName {
 						continue
 					}
-					content, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName)
+					content, _, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName)
 					if i%2 == 0 {
 						Expect(err).To(HaveOccurred(), "Expected file not found") // File should not exist
 					} else {

test/e2e/upgrade/upgrade.go

@@ -29,7 +29,6 @@ import (
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/util/kibishii"
-
 	. "github.com/vmware-tanzu/velero/test/util/providers"
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
@@ -256,7 +255,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 
 			By(fmt.Sprintf("Verify workload %s after restore ", upgradeNamespace), func() {
 				Expect(KibishiiVerifyAfterRestore(*veleroCfg.ClientToInstallVelero, upgradeNamespace,
-					oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+					oneHourTimeout, DefaultKibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 			})
 		})
 	})

test/util/csi/common.go

@@ -21,14 +21,12 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
@@ -128,6 +126,7 @@ func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, err
 	}
 	return snapshotHandleList, nil
 }
+
 func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
 	pvcList, err := GetPvcByPVCName(context.Background(), namespace, podName)
 	if err != nil {

test/util/k8s/common.go

@@ -104,7 +104,6 @@ func GetPvcByPVCName(ctx context.Context, namespace, pvcName string) ([]string,
 		Args: []string{"{print $1}"},
 	}
 	cmds = append(cmds, cmd)
-
 	return common.GetListByCmdPipes(ctx, cmds)
 }
 
@@ -279,15 +278,30 @@ func CreateFileToPod(ctx context.Context, namespace, podName, containerName, vol
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
 	return cmd.Run()
 }
-func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, error) {
+func FileExistInPV(ctx context.Context, namespace, podName, containerName, volume, filename string) (bool, error) {
+	stdout, stderr, err := ReadFileFromPodVolume(ctx, namespace, podName, containerName, volume, filename)
+
+	output := fmt.Sprintf("%s:%s", stdout, stderr)
+	if strings.Contains(output, fmt.Sprintf("/%s/%s: No such file or directory", volume, filename)) {
+		return false, nil
+	} else {
+		if err == nil {
+			return true, nil
+		} else {
+			return false, errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s",
+				filename, volume, podName, namespace))
+		}
+	}
+}
+func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, string, error) {
 	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
 		"--", "cat", fmt.Sprintf("/%s/%s", volume, filename)}
 	cmd := exec.CommandContext(ctx, "kubectl", arg...)
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
 	stdout, stderr, err := veleroexec.RunCommand(cmd)
-	fmt.Print(stdout)
-	fmt.Print(stderr)
-	return stdout, err
+	fmt.Printf("stdout: %s\n", stdout)
+	fmt.Printf("stderr: %s\n", stderr)
+	return stdout, stderr, err
 }
 
 func RunCommand(cmdName string, arg []string) string {

test/util/k8s/persistentvolumes.go

@@ -22,10 +22,9 @@ import (
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/client-go/util/retry"
-
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
 )
 
 func CreatePersistentVolume(client TestClient, name string) (*corev1.PersistentVolume, error) {
@@ -93,3 +92,16 @@ func ClearClaimRefForFailedPVs(ctx context.Context, client TestClient) error {
 
 	return nil
 }
+
+func GetAllPVNames(ctx context.Context, client TestClient) ([]string, error) {
+	var pvNameList []string
+	pvList, err := client.ClientGo.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to List PV")
+	}
+
+	for _, pvName := range pvList.Items {
+		pvNameList = append(pvNameList, pvName.Name)
+	}
+	return pvNameList, nil
+}