diff --git a/changelogs/unreleased/6923-reasonerjt b/changelogs/unreleased/6923-reasonerjt new file mode 100644 index 0000000000..2a9c2aabdd --- /dev/null +++ b/changelogs/unreleased/6923-reasonerjt @@ -0,0 +1 @@ +Bump up aws sdk to aws-sdk-go-v2 diff --git a/go.mod b/go.mod index 51fd1b5540..75e59fd871 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,11 @@ require ( github.com/Azure/go-autorest/autorest v0.11.27 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.3.0 - github.com/aws/aws-sdk-go v1.44.256 + github.com/aws/aws-sdk-go-v2 v1.21.0 + github.com/aws/aws-sdk-go-v2/config v1.18.42 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.87 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.123.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0 github.com/bombsimon/logrusr/v3 v3.0.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/fatih/color v1.15.0 @@ -77,6 +81,21 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.40 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.14.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect diff --git a/go.sum b/go.sum index 39ddf41071..754c7f6f2f 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,46 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= -github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= +github.com/aws/aws-sdk-go-v2/config v1.18.42 h1:28jHROB27xZwU0CB88giDSjz7M1Sba3olb5JBGwina8= +github.com/aws/aws-sdk-go-v2/config v1.18.42/go.mod h1:4AZM3nMMxwlG+eZlxvBKqwVbkDLlnN2a4UGTL6HjaZI= +github.com/aws/aws-sdk-go-v2/credentials v1.13.40 h1:s8yOkDh+5b1jUDhMBtngF6zKWLDs84chUk2Vk0c38Og= +github.com/aws/aws-sdk-go-v2/credentials v1.13.40/go.mod h1:VtEHVAAqDWASwdOqj/1huyT6uHbs5s8FUHfDQdky/Rs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.87 h1:e20ZrsgDPUXqg8+rZVuPwNSp6yniUN2Yr2tzFZ+Yvl0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.87/go.mod h1:0i0TAT6W+5i48QTlDU2KmY6U2hBZeY/LCP0wktya2oc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 h1:g+qlObJH4Kn4n21g69DjspU0hKTjWtq7naZ9OLCv0ew= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.123.0 h1:AkjjaINgZQlz3valIVmlrs18jsl+fzYuoxED8oOVrYo= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.123.0/go.mod h1:0FhI2Rzcv5BNM3dNnbcCx2qa2naFZoAidJi11cQgzL0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0 h1:wl5dxN1NONhTDQD9uaEvNsDRX29cBmGED/nl0jkWlt4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= +github.com/aws/aws-sdk-go-v2/service/sso v1.14.1 h1:YkNzx1RLS0F5qdf9v1Q8Cuv9NXCL2TkosOxhzlUPV64= +github.com/aws/aws-sdk-go-v2/service/sso v1.14.1/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.1 h1:8lKOidPkmSmfUtiTgtdXWgaKItCZ/g75/jEk6Ql6GsA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.1/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= +github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 h1:s4bioTgjSFRwOoyEFzAVCmFmoowBgjTR8gkrF/sQ4wk= +github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -368,6 +406,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -941,7 +980,6 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1055,7 +1093,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= @@ -1064,7 +1101,6 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1076,7 +1112,11 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +<<<<<<< HEAD golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +======= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +>>>>>>> 04446a4f (Bump up aws sdk to aws-sdk-go-v2) golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/repository/config/aws.go b/pkg/repository/config/aws.go index cc4e14f598..ee229949e1 100644 --- a/pkg/repository/config/aws.go +++ b/pkg/repository/config/aws.go @@ -21,12 +21,10 @@ import ( "context" "os" - goerr "errors" - - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/endpoints" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/pkg/errors" ) @@ -73,27 +71,28 @@ func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) { // GetS3Credentials gets the S3 credential values according to the information // of the provided config or the system's environment variables -func GetS3Credentials(config map[string]string) (*credentials.Value, error) { +func GetS3Credentials(config map[string]string) (*aws.Credentials, error) { if os.Getenv(awsRoleEnvVar) != "" { return nil, nil } - opts := session.Options{} + var opts []func(*awsconfig.LoadOptions) error credentialsFile := config[CredentialsFileKey] if credentialsFile == "" { credentialsFile = os.Getenv(awsCredentialsFileEnvVar) } if credentialsFile != "" { - opts.SharedConfigFiles = append(opts.SharedConfigFiles, credentialsFile) - opts.SharedConfigState = session.SharedConfigEnable + opts = append(opts, awsconfig.WithSharedCredentialsFiles([]string{credentialsFile}), + // To support the existing use case where config file is passed + // as credentials of a BSL + awsconfig.WithSharedConfigFiles([]string{credentialsFile})) } - sess, err := session.NewSessionWithOptions(opts) + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) if err != nil { return nil, err } - - creds, err := sess.Config.Credentials.Get() + creds, err := cfg.Credentials.Retrieve(context.Background()) return &creds, err } @@ -101,33 +100,17 @@ func GetS3Credentials(config map[string]string) (*credentials.Value, error) { // GetAWSBucketRegion returns the AWS region that a bucket is in, or an error // if the region cannot be determined. func GetAWSBucketRegion(bucket string) (string, error) { - sess, err := session.NewSession() + cfg, err := awsconfig.LoadDefaultConfig(context.Background()) if err != nil { return "", errors.WithStack(err) } - - var region string - var requestErrs []error - - for _, partition := range endpoints.DefaultPartitions() { - for regionHint := range partition.Regions() { - region, err = s3manager.GetBucketRegion(context.Background(), sess, bucket, regionHint) - if err != nil { - requestErrs = append(requestErrs, errors.Wrapf(err, "error to get region with hint %s", regionHint)) - } - - // we only need to try a single region hint per partition, so break after the first - break - } - - if region != "" { - return region, nil - } + client := s3.NewFromConfig(cfg) + region, err := s3manager.GetBucketRegion(context.Background(), client, bucket) + if err != nil { + return "", errors.WithStack(err) } - - if requestErrs == nil { - return "", errors.Errorf("unable to determine region by bucket %s", bucket) - } else { - return "", errors.Wrapf(goerr.Join(requestErrs...), "error to get region by bucket %s", bucket) + if region == "" { + return "", errors.New("unable to determine bucket's region") } + return region, nil } diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index b98586fdb8..6d95f870e5 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -433,7 +433,7 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr if credValue != nil { result[udmrepo.StoreOptionS3KeyID] = credValue.AccessKeyID - result[udmrepo.StoreOptionS3Provider] = credValue.ProviderName + result[udmrepo.StoreOptionS3Provider] = credValue.Source result[udmrepo.StoreOptionS3SecretKey] = credValue.SecretAccessKey result[udmrepo.StoreOptionS3Token] = credValue.SessionToken } diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index 74cdc74b22..fb2639a83e 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -22,7 +22,7 @@ import ( "errors" "testing" - awscredentials "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/kopia/kopia/repo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -45,7 +45,7 @@ func TestGetStorageCredentials(t *testing.T) { credFileStore *credmock.FileStore credStoreError error credStorePath string - getS3Credentials func(map[string]string) (*awscredentials.Value, error) + getS3Credentials func(map[string]string) (*aws.Credentials, error) getGCPCredentials func(map[string]string) string expected map[string]string expectedErr string @@ -89,8 +89,8 @@ func TestGetStorageCredentials(t *testing.T) { }, }, }, - getS3Credentials: func(config map[string]string) (*awscredentials.Value, error) { - return &awscredentials.Value{ + getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { + return &aws.Credentials{ AccessKeyID: "from: " + config["credentialsFile"], }, nil }, @@ -115,8 +115,8 @@ func TestGetStorageCredentials(t *testing.T) { }, credFileStore: new(credmock.FileStore), credStorePath: "credentials-from-credential-key", - getS3Credentials: func(config map[string]string) (*awscredentials.Value, error) { - return &awscredentials.Value{ + getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { + return &aws.Credentials{ AccessKeyID: "from: " + config["credentialsFile"], }, nil }, @@ -138,7 +138,7 @@ func TestGetStorageCredentials(t *testing.T) { }, }, }, - getS3Credentials: func(config map[string]string) (*awscredentials.Value, error) { + getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { return nil, errors.New("fake error") }, credFileStore: new(credmock.FileStore), @@ -153,7 +153,7 @@ func TestGetStorageCredentials(t *testing.T) { Config: map[string]string{}, }, }, - getS3Credentials: func(config map[string]string) (*awscredentials.Value, error) { + getS3Credentials: func(config map[string]string) (*aws.Credentials, error) { return nil, nil }, credFileStore: new(credmock.FileStore), diff --git a/test/util/providers/aws_utils.go b/test/util/providers/aws_utils.go index f9dd597acd..9f70a6708e 100644 --- a/test/util/providers/aws_utils.go +++ b/test/util/providers/aws_utils.go @@ -17,19 +17,24 @@ limitations under the License. package providers import ( + "crypto/tls" + "crypto/x509" "fmt" + "net/http" "net/url" "os" - "strconv" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/endpoints" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/config" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/pkg/errors" "golang.org/x/net/context" @@ -55,10 +60,26 @@ const ( enableSharedConfigKey = "enableSharedConfig" ) -func newSessionOptions(config aws.Config, profile, caCert, credentialsFile, enableSharedConfig string) (session.Options, error) { - sessionOptions := session.Options{Config: config, Profile: profile} - if caCert != "" { - sessionOptions.CustomCABundle = strings.NewReader(caCert) +func newAWSConfig(region, profile, credentialsFile string, insecureSkipTLSVerify bool, caCert string) (aws.Config, error) { + empty := aws.Config{} + client := awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) { + if len(caCert) > 0 { + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM([]byte(caCert)) + if tr.TLSClientConfig == nil { + tr.TLSClientConfig = &tls.Config{ + RootCAs: caCertPool, + } + } else { + tr.TLSClientConfig.RootCAs = caCertPool + } + } + tr.TLSClientConfig.InsecureSkipVerify = insecureSkipTLSVerify + }) + opts := []func(*config.LoadOptions) error{ + config.WithRegion(region), + config.WithSharedConfigProfile(profile), + config.WithHTTPClient(client), } if credentialsFile == "" && os.Getenv("AWS_SHARED_CREDENTIALS_FILE") != "" { @@ -68,104 +89,60 @@ func newSessionOptions(config aws.Config, profile, caCert, credentialsFile, enab if credentialsFile != "" { if _, err := os.Stat(credentialsFile); err != nil { if os.IsNotExist(err) { - return session.Options{}, errors.Wrapf(err, "provided credentialsFile does not exist") + return empty, errors.Wrapf(err, "provided credentialsFile does not exist") } - return session.Options{}, errors.Wrapf(err, "could not get credentialsFile info") + return empty, errors.Wrapf(err, "could not get credentialsFile info") } - sessionOptions.SharedConfigFiles = append(sessionOptions.SharedConfigFiles, credentialsFile) - sessionOptions.SharedConfigState = session.SharedConfigEnable + opts = append(opts, config.WithSharedCredentialsFiles([]string{credentialsFile}), + config.WithSharedConfigFiles([]string{credentialsFile})) } - return sessionOptions, nil -} - -// takes AWS session options to create a new session -func getSession(options session.Options) (*session.Session, error) { - sess, err := session.NewSessionWithOptions(options) + awsConfig, err := config.LoadDefaultConfig(context.Background(), opts...) if err != nil { - return nil, errors.WithStack(err) + return empty, errors.Wrapf(err, "could not load config") } - - if _, err := sess.Config.Credentials.Get(); err != nil { - return nil, errors.WithStack(err) + if _, err := awsConfig.Credentials.Retrieve(context.Background()); err != nil { + return empty, errors.WithStack(err) } - return sess, nil + return awsConfig, nil } -// GetBucketRegion returns the AWS region that a bucket is in, or an error -// if the region cannot be determined. -func GetBucketRegion(bucket string) (string, error) { - var region string - - session, err := session.NewSession() - if err != nil { - return "", errors.WithStack(err) +func newS3Client(cfg aws.Config, url string, forcePathStyle bool) (*s3.Client, error) { + opts := []func(*s3.Options){ + func(o *s3.Options) { + o.UsePathStyle = forcePathStyle + }, } - - for _, partition := range endpoints.DefaultPartitions() { - for regionHint := range partition.Regions() { - region, _ = s3manager.GetBucketRegion(context.Background(), session, bucket, regionHint) - - // we only need to try a single region hint per partition, so break after the first - break - } - - if region != "" { - return region, nil + if url != "" { + if !IsValidS3URLScheme(url) { + return nil, errors.Errorf("Invalid s3 url %s, URL must be valid according to https://golang.org/pkg/net/url/#Parse and start with http:// or https://", url) } + opts = append(opts, func(o *s3.Options) { + o.BaseEndpoint = aws.String(url) + }) } - return "", errors.New("unable to determine bucket's region") + return s3.NewFromConfig(cfg, opts...), nil } -func (s AWSStorage) CreateSession(credentialProfile, credentialsFile, enableSharedConfig, caCert, bucket, bslPrefix, bslConfig string) (*session.Session, error) { - var err error - config := flag.NewMap() - config.Set(bslConfig) - region := config.Data()["region"] - objectsInput := s3.ListObjectsV2Input{} - objectsInput.Bucket = aws.String(bucket) - objectsInput.Delimiter = aws.String("/") - s3URL := "" - s3ForcePathStyleVal := "" - s3ForcePathStyle := false - - if s3ForcePathStyleVal != "" { - if s3ForcePathStyle, err = strconv.ParseBool(s3ForcePathStyleVal); err != nil { - return nil, errors.Wrapf(err, "could not parse %s (expected bool)", s3ForcePathStyleKey) - } - } - - // AWS (not an alternate S3-compatible API) and region not - // explicitly specified: determine the bucket's region - if s3URL == "" && region == "" { - var err error - - region, err = GetBucketRegion(bucket) - if err != nil { - return nil, err - } - } +// GetBucketRegion returns the AWS region that a bucket is in, or an error +// if the region cannot be determined. +func GetBucketRegion(bucket string) (string, error) { - serverConfig, err := newAWSConfig(s3URL, region, s3ForcePathStyle) + cfg, err := config.LoadDefaultConfig(context.Background()) if err != nil { - return nil, err + return "", errors.WithStack(err) } - - sessionOptions, err := newSessionOptions(*serverConfig, credentialProfile, caCert, credentialsFile, enableSharedConfig) + client := s3.NewFromConfig(cfg) + region, err := s3manager.GetBucketRegion(context.Background(), client, bucket) if err != nil { - return nil, err + return "", errors.WithStack(err) } - - serverSession, err := getSession(sessionOptions) - - if err != nil { - fmt.Println(err) - return nil, err + if region == "" { + return "", errors.New("unable to determine bucket's region") } - - return serverSession, nil + return region, nil } // IsValidS3URLScheme returns true if the scheme is http:// or https:// @@ -180,34 +157,9 @@ func IsValidS3URLScheme(s3URL string) bool { } return true } -func newAWSConfig(url, region string, forcePathStyle bool) (*aws.Config, error) { - awsConfig := aws.NewConfig(). - WithRegion(region). - WithS3ForcePathStyle(forcePathStyle) - - if url != "" { - if !IsValidS3URLScheme(url) { - return nil, errors.Errorf("Invalid s3 url %s, URL must be valid according to https://golang.org/pkg/net/url/#Parse and start with http:// or https://", url) - } - - awsConfig = awsConfig.WithEndpointResolver( - endpoints.ResolverFunc(func(service, region string, optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) { - if service == s3.EndpointsID { - return endpoints.ResolvedEndpoint{ - URL: url, - }, nil - } - - return endpoints.DefaultResolver().EndpointFor(service, region, optFns...) - }), - ) - } - - return awsConfig, nil -} -func (s AWSStorage) ListItems(client *s3.S3, objectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { - res, err := client.ListObjectsV2(objectsV2Input) +func (s AWSStorage) ListItems(client *s3.Client, objectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + res, err := client.ListObjectsV2(context.Background(), objectsV2Input) if err != nil { return nil, err } @@ -215,14 +167,15 @@ func (s AWSStorage) ListItems(client *s3.S3, objectsV2Input *s3.ListObjectsV2Inp return res, nil } -func (s AWSStorage) DeleteItem(client *s3.S3, deleteObjectV2Input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { - res, err := client.DeleteObject(deleteObjectV2Input) +func (s AWSStorage) DeleteItem(client *s3.Client, deleteObjectV2Input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + res, err := client.DeleteObject(context.Background(), deleteObjectV2Input) if err != nil { return nil, err } fmt.Println(res) return res, nil } + func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { config := flag.NewMap() config.Set(bslConfig) @@ -235,38 +188,42 @@ func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix } var err error - var s3Config *aws.Config - var sess *session.Session + var s3Config aws.Config + var s3Client *s3.Client region := config.Data()["region"] s3url := "" - + if region == "" { + region, err = GetBucketRegion(bslBucket) + if err != nil { + return false, errors.Wrapf(err, "failed to get region for bucket %s", bslBucket) + } + } if region == "minio" { s3url = config.Data()["s3Url"] - s3Config = &aws.Config{ - Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), - Endpoint: aws.String(s3url), - Region: aws.String(region), - DisableSSL: aws.Bool(true), - S3ForcePathStyle: aws.Bool(true), + s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, true, "") + if err != nil { + return false, errors.Wrapf(err, "Failed to create AWS config of region %s", region) } - sess, err = session.NewSession(s3Config) + s3Client, err = newS3Client(s3Config, s3url, true) } else { - sess, err = s.CreateSession("", cloudCredentialsFile, "false", "", "", "", bslConfig) + s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, false, "") + if err != nil { + return false, errors.Wrapf(err, "Failed to create AWS config of region %s", region) + } + s3Client, err = newS3Client(s3Config, s3url, true) } - if err != nil { - return false, errors.Wrapf(err, fmt.Sprintf("Failed to create AWS session of region %s", region)) + return false, errors.Wrapf(err, "failed to create S3 client of region %s", region) } - svc := s3.New(sess) - bucketObjects, err := s.ListItems(svc, &objectsInput) + bucketObjects, err := s.ListItems(s3Client, &objectsInput) if err != nil { fmt.Println("Couldn't retrieve bucket items!") return false, errors.Wrapf(err, "Couldn't retrieve bucket items") } for _, item := range bucketObjects.Contents { - fmt.Println(*item) + fmt.Println(item) } var backupNameInStorage string for _, item := range bucketObjects.CommonPrefixes { @@ -288,50 +245,64 @@ func (s AWSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPr config.Set(bslConfig) var err error - var sess *session.Session + var s3Config aws.Config + var s3Client *s3.Client region := config.Data()["region"] s3url := "" - - s3Config := &aws.Config{ - Region: aws.String(region), - Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + if region == "" { + region, err = GetBucketRegion(bslBucket) + if err != nil { + return errors.Wrapf(err, "failed to get region for bucket %s", bslBucket) + } } if region == "minio" { s3url = config.Data()["s3Url"] - s3Config = &aws.Config{ - Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), - Endpoint: aws.String(s3url), - Region: aws.String(region), - DisableSSL: aws.Bool(true), - S3ForcePathStyle: aws.Bool(true), + s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, true, "") + if err != nil { + return errors.Wrapf(err, "Failed to create AWS config of region %s", region) } - sess, err = session.NewSession(s3Config) + s3Client, err = newS3Client(s3Config, s3url, true) } else { - sess, err = s.CreateSession("", cloudCredentialsFile, "false", "", "", "", bslConfig) + s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, false, "") + if err != nil { + return errors.Wrapf(err, "Failed to create AWS config of region %s", region) + } + s3Client, err = newS3Client(s3Config, s3url, false) } if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Failed to create AWS session of region %s", region)) + return errors.Wrapf(err, "Failed to create S3 client of region %s", region) } - - svc := s3.New(sess) - fullPrefix := strings.Trim(bslPrefix, "/") + "/" + strings.Trim(backupObject, "/") + "/" - iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{ + listInput := &s3.ListObjectsV2Input{ Bucket: aws.String(bslBucket), Prefix: aws.String(fullPrefix), + } + // list all keys + var objectIds []s3types.ObjectIdentifier + p := s3.NewListObjectsV2Paginator(s3Client, listInput) + for p.HasMorePages() { + page, err := p.NextPage(context.Background()) + if err != nil { + return errors.Wrapf(err, "failed to list objects in bucket %s", bslBucket) + } + for _, obj := range page.Contents { + objectIds = append(objectIds, s3types.ObjectIdentifier{Key: aws.String(*obj.Key)}) + } + } + _, err = s3Client.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{ + Bucket: aws.String(bslBucket), + Delete: &s3types.Delete{Objects: objectIds}, }) - - if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil { - return errors.Wrapf(err, "Fail to delete object") + if err != nil { + return errors.Wrapf(err, "failed to delete objects from bucket %s", bslBucket) } fmt.Printf("Deleted object(s) from bucket: %s %s \n", bslBucket, fullPrefix) return nil } func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck test.SnapshotCheckPoint) error { - config := flag.NewMap() config.Set(bslConfig) region := config.Data()["region"] @@ -339,26 +310,22 @@ func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj if region == "minio" { return errors.New("No snapshot for Minio provider") } - sess, err := s.CreateSession("", cloudCredentialsFile, "false", "", "", "", bslConfig) - + cfg, err := newAWSConfig(region, "", cloudCredentialsFile, false, "") if err != nil { - fmt.Printf("Fail to create session with profile %s and config %s", cloudCredentialsFile, bslConfig) - return errors.Wrapf(err, "Fail to create session with profile %s and config %s", cloudCredentialsFile, bslConfig) + return errors.Wrapf(err, "Failed to create AWS config of region %s", region) } - svc := ec2.New(sess) - params := &ec2.DescribeSnapshotsInput{ - OwnerIds: []*string{aws.String("self")}, - Filters: []*ec2.Filter{ + ec2Client := ec2.NewFromConfig(cfg) + input := &ec2.DescribeSnapshotsInput{ + OwnerIds: []string{"self"}, + Filters: []ec2types.Filter{ { - Name: aws.String("tag:velero.io/backup"), - Values: []*string{ - aws.String(backupObject), - }, + Name: aws.String("tag:velero.io/backup"), + Values: []string{backupObject}, }, }, } - result, err := svc.DescribeSnapshots(params) + result, err := ec2Client.DescribeSnapshots(context.Background(), input) if err != nil { fmt.Println(err) } @@ -386,52 +353,46 @@ func (s AWSStorage) GetMinioBucketSize(cloudCredentialsFile, bslBucket, bslPrefi config := flag.NewMap() config.Set(bslConfig) region := config.Data()["region"] - s3url := config.Data()["s3Url"] - s3Config := &aws.Config{ - Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), - Endpoint: aws.String(s3url), - Region: aws.String(region), - DisableSSL: aws.Bool(true), - S3ForcePathStyle: aws.Bool(true), - } if region != "minio" { return 0, errors.New("it only supported by minio") } - sess, err := session.NewSession(s3Config) + s3url := config.Data()["s3Url"] + s3Config, err := newAWSConfig(region, "", cloudCredentialsFile, true, "") + if err != nil { + return 0, errors.Wrapf(err, "failed to create AWS config of region %s", region) + } + s3Client, err := newS3Client(s3Config, s3url, true) if err != nil { - return 0, errors.Wrapf(err, "Error create config session") + return 0, errors.Wrapf(err, "failed to create S3 client of region %s", region) } + /* + s3Config := &aws.Config{ + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + Endpoint: aws.String(s3url), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + */ - svc := s3.New(sess) var totalSize int64 - var continuationToken *string // Paginate through objects in the bucket objectsInput := &s3.ListObjectsV2Input{ - Bucket: aws.String(bslBucket), - ContinuationToken: continuationToken, + Bucket: aws.String(bslBucket), } if bslPrefix != "" { objectsInput.Prefix = aws.String(bslPrefix) } - for { - resp, err := svc.ListObjectsV2(objectsInput) + p := s3.NewListObjectsV2Paginator(s3Client, objectsInput) + for p.HasMorePages() { + page, err := p.NextPage(context.Background()) if err != nil { - return 0, errors.Wrapf(err, "Error list objects") + return 0, errors.Wrapf(err, "failed to list objects in bucket %s", bslBucket) } - - // Process objects in the current response - for _, obj := range resp.Contents { - totalSize += *obj.Size + for _, obj := range page.Contents { + totalSize += obj.Size } - - // Check if there are more objects to retrieve - if !*resp.IsTruncated { - break - } - - // Set the continuation token for the next page - continuationToken = resp.NextContinuationToken } return totalSize, nil }