feat: awss3 DownloadFiles use backoff

88labs · Aug 31, 2023 · efea36e · efea36e
1 parent 1ce7edc
commit efea36e
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 4 deletions.
diff --git a/aws/awss3/awss3.go b/aws/awss3/awss3.go
@@ -15,6 +15,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
+
 	"github.com/88labs/go-utils/aws/awss3/options/s3list"
 
 	"github.com/88labs/go-utils/aws/awss3/options/s3head"
@@ -326,10 +328,16 @@ func DownloadFiles(ctx context.Context, region awsconfig.Region, bucketName Buck
 			return nil, err
 		}
 		eg.Go(func() error {
-			if _, err := downloader.Download(ctx, f, &s3.GetObjectInput{
-				Bucket: bucketName.AWSString(),
-				Key:    s3Key.AWSString(),
-			}); err != nil {
+			b := backoff.WithContext(backoff.NewExponentialBackOff(), ctx)
+			if err := backoff.Retry(func() error {
+				if _, err := downloader.Download(ctx, f, &s3.GetObjectInput{
+					Bucket: bucketName.AWSString(),
+					Key:    s3Key.AWSString(),
+				}); err != nil {
+					return err
+				}
+				return nil
+			}, b); err != nil {
 				return err
 			}
 			return nil

diff --git a/aws/go.mod b/aws/go.mod
@@ -39,6 +39,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect

diff --git a/aws/go.sum b/aws/go.sum
@@ -56,6 +56,8 @@ github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
 github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/bxcodec/faker/v3 v3.8.1 h1:qO/Xq19V6uHt2xujwpaetgKhraGCapqY2CRWGD/SqcM=
 github.com/bxcodec/faker/v3 v3.8.1/go.mod h1:DdSDccxF5msjFo5aO4vrobRQ8nIApg8kq3QWPEQD6+o=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=