Flaky unit test - fails to update DRCluster manifest work

Signed-off-by: Elena Gershkovich <elenage@il.ibm.com>

controllers/drcluster_controller_test.go

@@ -22,6 +22,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
@@ -220,17 +221,22 @@ func updateDRClusterManifestWorkStatus(clusterNamespace string) {
 		},
 	}
 
-	mw.Status = DRClusterStatusConditions
+	retryErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		var err error
+
+		err = apiReader.Get(context.TODO(), manifestLookupKey, mw)
+		if err != nil {
+			return err
+		}
 
-	err := k8sClient.Status().Update(context.TODO(), mw)
-	if err != nil {
-		// try again
-		Expect(apiReader.Get(context.TODO(), manifestLookupKey, mw)).NotTo(HaveOccurred())
 		mw.Status = DRClusterStatusConditions
+
 		err = k8sClient.Status().Update(context.TODO(), mw)
-	}
 
-	Expect(err).NotTo(HaveOccurred())
+		return err
+	})
+
+	Expect(retryErr).NotTo(HaveOccurred())
 }
 
 var _ = Describe("DRClusterController", func() {

controllers/util/mw_util.go

@@ -20,6 +20,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	dto "github.com/prometheus/client_model/go"
@@ -502,11 +503,28 @@ func (mwu *MWUtil) createOrUpdateManifestWork(
 	}
 
 	if !reflect.DeepEqual(foundMW.Spec, mw.Spec) {
-		mw.Spec.DeepCopyInto(&foundMW.Spec)
-
 		mwu.Log.Info("ManifestWork exists.", "name", mw.Name, "namespace", foundMW.Namespace)
 
-		return mwu.Client.Update(mwu.Ctx, foundMW)
+		retryErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+			var err error
+
+			err = mwu.Client.Get(mwu.Ctx,
+				types.NamespacedName{Name: mw.Name, Namespace: managedClusternamespace},
+				foundMW)
+			if err != nil {
+				return err
+			}
+
+			mw.Spec.DeepCopyInto(&foundMW.Spec)
+
+			err = mwu.Client.Update(mwu.Ctx, foundMW)
+
+			return err
+		})
+
+		if retryErr != nil {
+			return retryErr
+		}
 	}
 
 	return nil