From 396692b54af9e488129c02e5d88eae5800daaac9 Mon Sep 17 00:00:00 2001 From: Luc Talatinian Date: Tue, 15 Oct 2024 10:47:00 -0400 Subject: [PATCH 1/3] add loadoptions to configure BaseEndpoint --- .../024e7efafa274001b8677eb90bd32260.json | 8 + config/load_options.go | 33 ++++ .../integrationtest/s3/endpoint_url_test.go | 187 ++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 .changelog/024e7efafa274001b8677eb90bd32260.json create mode 100644 service/internal/integrationtest/s3/endpoint_url_test.go diff --git a/.changelog/024e7efafa274001b8677eb90bd32260.json b/.changelog/024e7efafa274001b8677eb90bd32260.json new file mode 100644 index 00000000000..190ccc8e98f --- /dev/null +++ b/.changelog/024e7efafa274001b8677eb90bd32260.json @@ -0,0 +1,8 @@ +{ + "id": "024e7efa-fa27-4001-b867-7eb90bd32260", + "type": "feature", + "description": "Adds the LoadOptions hook `WithBaseEndpoint` for setting global endpoint override in-code.", + "modules": [ + "config" + ] +} diff --git a/config/load_options.go b/config/load_options.go index 5f643977b00..dc6c7d29a83 100644 --- a/config/load_options.go +++ b/config/load_options.go @@ -217,6 +217,10 @@ type LoadOptions struct { S3DisableExpressAuth *bool AccountIDEndpointMode aws.AccountIDEndpointMode + + // Service endpoint override. This value is not necessarily final and is + // passed to the service's EndpointResolverV2 for further delegation. + BaseEndpoint string } func (o LoadOptions) getDefaultsMode(ctx context.Context) (aws.DefaultsMode, bool, error) { @@ -284,6 +288,19 @@ func (o LoadOptions) getAccountIDEndpointMode(ctx context.Context) (aws.AccountI return o.AccountIDEndpointMode, len(o.AccountIDEndpointMode) > 0, nil } +func (o LoadOptions) getBaseEndpoint(context.Context) (string, bool, error) { + return o.BaseEndpoint, o.BaseEndpoint != "", nil +} + +// GetServiceBaseEndpoint satisfies (internal/configsources).ServiceBaseEndpointProvider. +// +// The sdkID value is unused because LoadOptions only supports setting a GLOBAL +// endpoint override. In-code, per-service endpoint overrides are performed via +// functional options in service client space. +func (o LoadOptions) GetServiceBaseEndpoint(context.Context, string) (string, bool, error) { + return o.BaseEndpoint, o.BaseEndpoint != "", nil +} + // WithRegion is a helper function to construct functional options // that sets Region on config's LoadOptions. Setting the region to // an empty string, will result in the region value being ignored. @@ -1139,3 +1156,19 @@ func WithS3DisableExpressAuth(v bool) LoadOptionsFunc { return nil } } + +// WithBaseEndpoint is a helper function to construct functional options that +// sets BaseEndpoint on config's LoadOptions. Empty values have no effect, and +// subsequent calls to this API override previous ones. +// +// This is an in-code setting, therefore, any value set using this hook takes +// precedence over and will override ALL environment and shared config +// directives that set endpoint URLs. Functional options on service clients +// have higher specificity, and functional options that modify the value of +// BaseEndpoint on a client will take precedence over this setting. +func WithBaseEndpoint(v string) LoadOptionsFunc { + return func(o *LoadOptions) error { + o.BaseEndpoint = v + return nil + } +} diff --git a/service/internal/integrationtest/s3/endpoint_url_test.go b/service/internal/integrationtest/s3/endpoint_url_test.go new file mode 100644 index 00000000000..8876770519f --- /dev/null +++ b/service/internal/integrationtest/s3/endpoint_url_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "context" + "os" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// From the SEP: +// Service-specific endpoint configuration MUST be resolved with an endpoint URL provider chain with the following precedence: +// - The value provided through code to an AWS SDK or tool via a command line +// parameter or a client or configuration constructor; for example the +// --endpoint-url command line parameter or the endpoint_url parameter +// provided to the Python SDK client. +// - The value provided by a service-specific environment variable. +// - The value provided by the global endpoint environment variable +// (AWS_ENDPOINT_URL). +// - The value provided by a service-specific parameter from a services +// definition section referenced in a profile in the shared configuration +// file. +// - The value provided by the global parameter from a profile in the shared +// configuration file. +// - The value resolved through the methods provided by the SDK or tool when +// no explicit endpoint URL is provided. + +func TestEndpointURL(t *testing.T) { + for name, tt := range map[string]struct { + Env map[string]string + SharedConfig string + LoadOpts []func(*config.LoadOptions) error + ClientOpts []func(*s3.Options) + Expect string + }{ + "no values": { + SharedConfig: ` +[default] +`, + Expect: "", + }, + + "precedence 0: in-code, set via s3.Options": { + Env: map[string]string{ + "AWS_ENDPOINT_URL": "https://global-env.com", + "AWS_ENDPOINT_URL_S3": "https://service-env.com", + }, + SharedConfig: ` +[default] +endpoint_url = https://global-cfg.com +services = service_cfg + +[services service_cfg] +s3 = + endpoint_url = https://service-cfg.com +`, + LoadOpts: []func(*config.LoadOptions) error{ + config.WithBaseEndpoint("https://loadopts.com"), + }, + ClientOpts: []func(*s3.Options){ + func(o *s3.Options) { + o.BaseEndpoint = aws.String("https://clientopts.com") + }, + }, + Expect: "https://clientopts.com", + }, + + "precedence 0: in-code, set via config.LoadOptions": { + Env: map[string]string{ + "AWS_ENDPOINT_URL": "https://global-env.com", + "AWS_ENDPOINT_URL_S3": "https://service-env.com", + }, + SharedConfig: ` + [default] + endpoint_url = https://global-cfg.com + services = service_cfg + + [services service_cfg] + s3 = + endpoint_url = https://service-cfg.com + `, + LoadOpts: []func(*config.LoadOptions) error{ + config.WithBaseEndpoint("https://loadopts.com"), + }, + Expect: "https://loadopts.com", + }, + + "precedence 1: service env": { + Env: map[string]string{ + "AWS_ENDPOINT_URL": "https://global-env.com", + "AWS_ENDPOINT_URL_S3": "https://service-env.com", + }, + SharedConfig: ` +[default] +endpoint_url = https://global-cfg.com +services = service_cfg + +[services service_cfg] +s3 = + endpoint_url = https://service-cfg.com +`, + Expect: "https://service-env.com", + }, + + "precedence 2: global env": { + Env: map[string]string{ + "AWS_ENDPOINT_URL": "https://global-env.com", + }, + SharedConfig: ` +[default] +endpoint_url = https://global-cfg.com +services = service_cfg + +[services service_cfg] +s3 = + endpoint_url = https://service-cfg.com +`, + Expect: "https://global-env.com", + }, + + "precedence 3: service cfg": { + SharedConfig: ` +[default] +endpoint_url = https://global-cfg.com +services = service_cfg + +[services service_cfg] +s3 = + endpoint_url = https://service-cfg.com +`, + Expect: "https://service-cfg.com", + }, + + "precedence 4: global cfg": { + SharedConfig: ` +[default] +endpoint_url = https://global-cfg.com +`, + Expect: "https://global-cfg.com", + }, + } { + t.Run(name, func(t *testing.T) { + reset, err := mockEnvironment(tt.Env, tt.SharedConfig) + if err != nil { + t.Fatalf("mock environment: %v", err) + } + defer reset() + + loadopts := append(tt.LoadOpts, + config.WithSharedConfigFiles([]string{"test_shared_config"})) + cfg, err := config.LoadDefaultConfig(context.Background(), loadopts...) + if err != nil { + t.Fatalf("load config: %v", err) + } + + svc := s3.NewFromConfig(cfg, tt.ClientOpts...) + actual := aws.ToString(svc.Options().BaseEndpoint) + if tt.Expect != actual { + t.Errorf("expect endpoint: %q != %q", tt.Expect, actual) + } + }) + } +} + +func mockEnvironment(env map[string]string, sharedCfg string) (func(), error) { + for k, v := range env { + os.Setenv(k, v) + } + f, err := os.Create("test_shared_config") + if err != nil { + return nil, err + } + if _, err := f.Write([]byte(sharedCfg)); err != nil { + return nil, err + } + + return func() { + for k := range env { + os.Unsetenv(k) + } + if err := os.Remove("test_shared_config"); err != nil { + panic(err) + } + }, nil +} From 5055d6a00af63158e9c9914a40aa2db05d569902 Mon Sep 17 00:00:00 2001 From: Luc Talatinian Date: Tue, 15 Oct 2024 14:36:59 -0400 Subject: [PATCH 2/3] add integration flag and fix package --- service/internal/integrationtest/s3/endpoint_url_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/service/internal/integrationtest/s3/endpoint_url_test.go b/service/internal/integrationtest/s3/endpoint_url_test.go index 8876770519f..b86c7ef71ff 100644 --- a/service/internal/integrationtest/s3/endpoint_url_test.go +++ b/service/internal/integrationtest/s3/endpoint_url_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package s3 import ( "context" From a9556b3a3b26195e33ffa256f75603a1bbb4c419 Mon Sep 17 00:00:00 2001 From: Luc Talatinian Date: Tue, 15 Oct 2024 14:49:20 -0400 Subject: [PATCH 3/3] i'm really struggling with this --- service/internal/integrationtest/s3/endpoint_url_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/internal/integrationtest/s3/endpoint_url_test.go b/service/internal/integrationtest/s3/endpoint_url_test.go index b86c7ef71ff..f220544e69f 100644 --- a/service/internal/integrationtest/s3/endpoint_url_test.go +++ b/service/internal/integrationtest/s3/endpoint_url_test.go @@ -30,7 +30,7 @@ import ( // - The value resolved through the methods provided by the SDK or tool when // no explicit endpoint URL is provided. -func TestEndpointURL(t *testing.T) { +func TestInteg_EndpointURL(t *testing.T) { for name, tt := range map[string]struct { Env map[string]string SharedConfig string