Retry Get on IsNotFound after Create AlreadyExists

Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com>

Use MinuteBackoff for retry

Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com>

changelogs/unreleased/6949-kaovilai

@@ -0,0 +1 @@
+On restore, retry Get when IsNotFound after Create fails with AlreadyExists

pkg/client/retry.go

@@ -18,12 +18,29 @@ package client
 
 import (
 	"context"
+	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/retry"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+// MinuteBackoff is a retry.DefaultBackoff that retries for at least 1 minute (60000ms)
+var MinuteBackoff = func() wait.Backoff {
+	mb := retry.DefaultBackoff
+	// TotalDuration = 0ms + 10ms + 50ms + 250ms + 1250ms + 6250ms + 31250ms + 60000ms = 99,060 ms > 1 minute
+	// 7 steps
+	mb.Steps = 7
+	mb.Cap = time.Minute
+	return mb
+}()
+
 func CreateRetryGenerateName(client kbclient.Client, ctx context.Context, obj kbclient.Object) error {
 	return CreateRetryGenerateNameWithFunc(obj, func() error {
 		return client.Create(ctx, obj, &kbclient.CreateOptions{})
@@ -42,3 +59,25 @@ func CreateRetryGenerateNameWithFunc(obj kbclient.Object, createFn func() error)
 		return createFn()
 	}
 }
+
+func GetRetriableWithCacheLister(lister cache.GenericNamespaceLister, name string, retriable func(error) bool) (runtime.Object, error) {
+	var clusterObj runtime.Object
+	getFunc := func() error {
+		var err error
+		clusterObj, err = lister.Get(name)
+		return err
+	}
+	err := retry.OnError(MinuteBackoff, retriable, getFunc)
+	return clusterObj, err
+}
+
+func GetRetriableWithDynamicClient(client Dynamic, name string, getOptions metav1.GetOptions, retriable func(error) bool) (*unstructured.Unstructured, error) {
+	var clusterObj *unstructured.Unstructured
+	getFunc := func() error {
+		var err error
+		clusterObj, err = client.Get(name, getOptions)
+		return err
+	}
+	err := retry.OnError(MinuteBackoff, retriable, getFunc)
+	return clusterObj, err
+}

pkg/restore/restore.go

@@ -1082,6 +1082,21 @@ func (ctx *restoreContext) getResource(groupResource schema.GroupResource, obj *
 	return u, nil
 }
 
+// getResource but with retry on retriable error
+func (ctx *restoreContext) getResourceWithRetries(groupResource schema.GroupResource, obj *unstructured.Unstructured, namespace, name string, retriable func(error) bool) (*unstructured.Unstructured, error) {
+	lister := ctx.getResourceLister(groupResource, obj, namespace)
+	clusterObj, err := client.GetRetriableWithCacheLister(lister, name, retriable)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting resource from lister for %s, %s/%s", groupResource, namespace, name)
+	}
+	u, ok := clusterObj.(*unstructured.Unstructured)
+	if !ok {
+		ctx.log.WithError(errors.WithStack(fmt.Errorf("expected *unstructured.Unstructured but got %T", u))).Error("unable to understand entry returned from client")
+		return nil, fmt.Errorf("expected *unstructured.Unstructured but got %T", u)
+	}
+	return u, nil
+}
+
 func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupResource schema.GroupResource, namespace string) (results.Result, results.Result, bool) {
 	warnings, errs := results.Result{}, results.Result{}
 	// itemExists bool is used to determine whether to include this item in the "wait for additional items" list
@@ -1529,9 +1544,9 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 		// if so, we will return the 'get' error.
 		// otherwise, we will return the original creation error.
 		if !ctx.disableInformerCache {
-			fromCluster, err = ctx.getResource(groupResource, obj, namespace, name)
+			fromCluster, err = ctx.getResourceWithRetries(groupResource, obj, namespace, name, apierrors.IsNotFound)
 		} else {
-			fromCluster, err = resourceClient.Get(name, metav1.GetOptions{})
+			fromCluster, err = client.GetRetriableWithDynamicClient(resourceClient, name, metav1.GetOptions{}, apierrors.IsNotFound)
 		}
 		if err != nil && isAlreadyExistsError {
 			ctx.log.Errorf("Error retrieving in-cluster version of %s: %v", kube.NamespaceAndName(obj), err)