From 041cb09ebe0fc6dc72d192e8fb97898913c47efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rold=C3=A1n=20Betancort?= Date: Wed, 20 Dec 2023 10:36:06 +0000 Subject: [PATCH] restore: improve progress report on retries --- internal/cmd/restorer.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/cmd/restorer.go b/internal/cmd/restorer.go index 0a173f52..f35daab3 100644 --- a/internal/cmd/restorer.go +++ b/internal/cmd/restorer.go @@ -27,6 +27,9 @@ const ( Fail ConflictStrategy = iota Skip Touch + + defaultBackoff = 50 * time.Millisecond + defaultMaxRetries = 10 ) var conflictStrategyMapping = map[string]ConflictStrategy{ @@ -299,7 +302,7 @@ func (r *restorer) commitStream(ctx context.Context, bulkImportClient v1.Experim // each batch will be committed independently. If a batch fails, it will be retried up to 10 times with a backoff. func (r *restorer) writeBatchesWithRetry(ctx context.Context, batches [][]*v1.Relationship) (uint64, uint64, error) { backoffInterval := backoff.NewExponentialBackOff() - backoffInterval.InitialInterval = 10 * time.Millisecond + backoffInterval.InitialInterval = defaultBackoff backoffInterval.MaxInterval = 2 * time.Second backoffInterval.MaxElapsedTime = 0 backoffInterval.Reset() @@ -314,17 +317,19 @@ func (r *restorer) writeBatchesWithRetry(ctx context.Context, batches [][]*v1.Re }) for { - // throttle the writes so we don't overwhelm the server - time.Sleep(backoffInterval.NextBackOff()) cancelCtx, cancel := context.WithTimeout(ctx, r.requestTimeout) _, err := r.client.WriteRelationships(cancelCtx, &v1.WriteRelationshipsRequest{Updates: updates}) cancel() - if isRetryableError(err) && currentRetries < 10 { + if isRetryableError(err) && currentRetries < defaultMaxRetries { + // throttle the writes so we don't overwhelm the server + bo := backoffInterval.NextBackOff() + r.bar.Describe(fmt.Sprintf("retrying write with backoff %s after error (attempt %d/%d)", bo, + currentRetries+1, defaultMaxRetries)) + time.Sleep(bo) currentRetries++ r.totalRetries++ totalRetries++ - r.bar.Describe(fmt.Sprintf("retrying after error (attempt %d)", currentRetries+1)) continue } if err != nil {