Allow simulation of abnormal VR valiadtion

New promoteOptions{} allows simulation of 2 issues:

- Validated failed: The VR will never complete, blocking deletion of the
  VR and VRG. With this fix deletion will succeed.

- Validated condition missing - running ramen 4.17 on system with older
  csi-addons that does not report the condition. In this case ramen
  continue normally, and deletion of the VR and VRG will be blocked.

We can extend this later to support more options.

Signed-off-by: Nir Soffer <nsoffer@redhat.com>

internal/controller/vrg_volrep_test.go

@@ -2155,17 +2155,22 @@ func (v *vrgTest) waitForVRCountToMatch(vrCount int) {
 }
 
 func (v *vrgTest) promoteVolReps() {
-	v.promoteVolRepsAndDo(func(index, count int) {
+	v.promoteVolRepsAndDo(promoteOptions{}, func(index, count int) {
 		// VRG should not be ready until last VolRep is ready.
 		v.verifyVRGStatusExpectation(index == count-1, vrgController.VRGConditionReasonReady)
 	})
 }
 
 func (v *vrgTest) promoteVolRepsWithoutVrgStatusCheck() {
-	v.promoteVolRepsAndDo(func(index, count int) {})
+	v.promoteVolRepsAndDo(promoteOptions{}, func(index, count int) {})
 }
 
-func (v *vrgTest) promoteVolRepsAndDo(do func(int, int)) {
+type promoteOptions struct {
+	ValidatedMissing bool
+	ValidatedFailed  bool
+}
+
+func (v *vrgTest) promoteVolRepsAndDo(options promoteOptions, do func(int, int)) {
 	By("Promoting VolumeReplication resources " + v.namespace)
 
 	volRepList := &volrep.VolumeReplicationList{}
@@ -2178,43 +2183,65 @@ func (v *vrgTest) promoteVolRepsAndDo(do func(int, int)) {
 	for index := range volRepList.Items {
 		volRep := volRepList.Items[index]
 
-		volRepStatus := volrep.VolumeReplicationStatus{
-			Conditions: []metav1.Condition{
-				// Since csi-addons >= 0.10.0
-				{
-					Type:               volrep.ConditionValidated,
-					Reason:             volrep.PrerequisiteMet,
-					ObservedGeneration: volRep.Generation,
-					Status:             metav1.ConditionTrue,
-					LastTransitionTime: metav1.NewTime(time.Now()),
-				},
-				// Since csi-addons < 0.10.0
-				{
-					Type:               volrep.ConditionCompleted,
-					Reason:             volrep.Promoted,
-					ObservedGeneration: volRep.Generation,
-					Status:             metav1.ConditionTrue,
-					LastTransitionTime: metav1.NewTime(time.Now()),
-				},
-				{
-					Type:               volrep.ConditionDegraded,
-					Reason:             volrep.Healthy,
-					ObservedGeneration: volRep.Generation,
-					Status:             metav1.ConditionFalse,
-					LastTransitionTime: metav1.NewTime(time.Now()),
-				},
-				{
-					Type:               volrep.ConditionResyncing,
-					Reason:             volrep.NotResyncing,
-					ObservedGeneration: volRep.Generation,
-					Status:             metav1.ConditionFalse,
-					LastTransitionTime: metav1.NewTime(time.Now()),
-				},
-			},
+		var conditions []metav1.Condition
+
+		// csi-addons < 0.10.0
+		completed := metav1.Condition{
+			Type:               volrep.ConditionCompleted,
+			Reason:             volrep.Promoted,
+			ObservedGeneration: volRep.Generation,
+			Status:             metav1.ConditionTrue,
+			LastTransitionTime: metav1.NewTime(time.Now()),
+		}
+		degraded := metav1.Condition{
+			Type:               volrep.ConditionDegraded,
+			Reason:             volrep.Healthy,
+			ObservedGeneration: volRep.Generation,
+			Status:             metav1.ConditionFalse,
+			LastTransitionTime: metav1.NewTime(time.Now()),
+		}
+		resyncing := metav1.Condition{
+			Type:               volrep.ConditionResyncing,
+			Reason:             volrep.NotResyncing,
+			ObservedGeneration: volRep.Generation,
+			Status:             metav1.ConditionFalse,
+			LastTransitionTime: metav1.NewTime(time.Now()),
 		}
+
+		// csi-addons >= 0.10.0
+		if !options.ValidatedMissing {
+			validated := metav1.Condition{
+				Type:               volrep.ConditionValidated,
+				Reason:             volrep.PrerequisiteNotMet,
+				ObservedGeneration: volRep.Generation,
+				Status:             metav1.ConditionFalse,
+				LastTransitionTime: metav1.NewTime(time.Now()),
+			}
+
+			// If validation failed, completed is also failed.
+			if options.ValidatedFailed {
+				validated.Status = metav1.ConditionFalse
+				validated.Reason = volrep.PrerequisiteNotMet
+				completed.Status = metav1.ConditionFalse
+				completed.Reason = volrep.FailedToPromote
+			}
+
+			conditions = append(conditions, validated)
+		}
+
+		conditions = append(conditions, completed, degraded, resyncing)
+
+		volRepStatus := volrep.VolumeReplicationStatus{Conditions: conditions}
 		volRepStatus.ObservedGeneration = volRep.Generation
-		volRepStatus.State = volrep.PrimaryState
-		volRepStatus.Message = "volume is marked primary"
+
+		if options.ValidatedFailed {
+			volRepStatus.State = volrep.UnknownState
+			volRepStatus.Message = "precondition failed ..."
+		} else {
+			volRepStatus.State = volrep.PrimaryState
+			volRepStatus.Message = "volume is marked primary"
+		}
+
 		volRep.Status = volRepStatus
 
 		err = k8sClient.Status().Update(context.TODO(), &volRep)
@@ -2224,8 +2251,13 @@ func (v *vrgTest) promoteVolRepsAndDo(do func(int, int)) {
 			Name:      volRep.Name,
 			Namespace: volRep.Namespace,
 		}
-		v.waitForVolRepCondition(volrepKey, volrep.ConditionCompleted, metav1.ConditionTrue)
-		v.waitForProtectedPVCs(volrepKey)
+
+		if options.ValidatedFailed {
+			v.waitForVolRepCondition(volrepKey, volrep.ConditionValidated, metav1.ConditionFalse)
+		} else {
+			v.waitForVolRepCondition(volrepKey, volrep.ConditionCompleted, metav1.ConditionTrue)
+			v.waitForProtectedPVCs(volrepKey)
+		}
 
 		do(index, len(volRepList.Items))
 	}