From d9e6e1a6e5ad2f1ecd97f7cbde7599b23e825573 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 7 Oct 2024 13:30:55 -0700 Subject: [PATCH 01/18] Reduce memory usage from aws-partitions --- gems/aws-partitions/CHANGELOG.md | 2 ++ gems/aws-partitions/lib/aws-partitions.rb | 1 + .../lib/aws-partitions/metadata.rb | 30 +++++++++++++++++++ .../lib/aws-partitions/partition_list.rb | 1 + gems/aws-sdk-core/CHANGELOG.md | 2 ++ gems/aws-sdk-core/aws-sdk-core.gemspec | 2 +- .../lib/aws-sdk-core/endpoints/matchers.rb | 17 ++++++----- .../aws-sdk-core/plugins/regional_endpoint.rb | 28 ++++++++++++----- .../lib/seahorse/client/plugins/endpoint.rb | 11 ++++--- 9 files changed, 71 insertions(+), 23 deletions(-) create mode 100644 gems/aws-partitions/lib/aws-partitions/metadata.rb diff --git a/gems/aws-partitions/CHANGELOG.md b/gems/aws-partitions/CHANGELOG.md index c7126120db5..573fa31393b 100644 --- a/gems/aws-partitions/CHANGELOG.md +++ b/gems/aws-partitions/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Add partition metadata module, allowing access without loading entire partitions.json. + 1.985.0 (2024-10-04) ------------------ diff --git a/gems/aws-partitions/lib/aws-partitions.rb b/gems/aws-partitions/lib/aws-partitions.rb index a389c828b47..5c06f1d3310 100644 --- a/gems/aws-partitions/lib/aws-partitions.rb +++ b/gems/aws-partitions/lib/aws-partitions.rb @@ -5,6 +5,7 @@ require_relative 'aws-partitions/partition_list' require_relative 'aws-partitions/region' require_relative 'aws-partitions/service' +require_relative 'aws-partitions/metadata' require 'json' diff --git a/gems/aws-partitions/lib/aws-partitions/metadata.rb b/gems/aws-partitions/lib/aws-partitions/metadata.rb new file mode 100644 index 00000000000..60021dfc12a --- /dev/null +++ b/gems/aws-partitions/lib/aws-partitions/metadata.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Aws + module Partitions + # @api private + module Metadata + class << self + def aws_partition(value) + partition = + partitions.find { |p| p['regions']&.fetch(value) } || + partitions.find { |p| value.match(p['regionRegex']) } || + partitions.find { |p| p['id'] == 'aws' } + + return nil unless partition + + partition['outputs'] + end + + def partitions + @partitions ||= default_partition_metadata + end + + def default_partition_metadata + path = File.expand_path('../../../partitions-metadata.json', __FILE__) + JSON.parse(File.read(path), freeze: true)['partitions'] + end + end + end + end +end \ No newline at end of file diff --git a/gems/aws-partitions/lib/aws-partitions/partition_list.rb b/gems/aws-partitions/lib/aws-partitions/partition_list.rb index 002d61751c4..2fa56d158e0 100644 --- a/gems/aws-partitions/lib/aws-partitions/partition_list.rb +++ b/gems/aws-partitions/lib/aws-partitions/partition_list.rb @@ -90,6 +90,7 @@ def build_metadata_regions(partition_name, metadata_regions, existing = nil) else regions[region_name] = Region.new( name: region_name, + description: region['description'], partition_name: partition_name, services: Set.new diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 92f50c28c31..d4a9ca22d46 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - reduce memory usage by not using legacy endpoint data unless required. + 3.209.1 (2024-09-25) ------------------ diff --git a/gems/aws-sdk-core/aws-sdk-core.gemspec b/gems/aws-sdk-core/aws-sdk-core.gemspec index 6b393f4bfd3..c27a2259d5a 100644 --- a/gems/aws-sdk-core/aws-sdk-core.gemspec +++ b/gems/aws-sdk-core/aws-sdk-core.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |spec| spec.files = Dir['LICENSE.txt', 'CHANGELOG.md', 'VERSION', 'lib/**/*.rb', 'sig/**/*.rbs', 'ca-bundle.crt'] spec.add_dependency('jmespath', '~> 1', '>= 1.6.1') # necessary for secure jmespath JSON parsing - spec.add_dependency('aws-partitions', '~> 1', '>= 1.651.0') # necessary for new endpoint resolution + spec.add_dependency('aws-partitions', '~> 1', '>= 1.986.0') # necessary for new endpoint resolution spec.add_dependency('aws-sigv4', '~> 1.9') # necessary for s3 express auth/native sigv4a support spec.add_dependency('aws-eventstream', '~> 1', '>= 1.3.0') # necessary for binary eventstream diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb b/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb index 542b0abb73c..020070f4c1e 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb @@ -94,14 +94,15 @@ def self.valid_host_label?(value, allow_sub_domains = false) # aws.partition(value: string) Option def self.aws_partition(value) - partition = - Aws::Partitions.find { |p| p.region?(value) } || - Aws::Partitions.find { |p| value.match(p.region_regex) } || - Aws::Partitions.find { |p| p.name == 'aws' } - - return nil unless partition - - partition.metadata + Aws::Partitions::Metadata.aws_partition(value) + # partition = + # Aws::Partitions.find { |p| p.region?(value) } || + # Aws::Partitions.find { |p| value.match(p.region_regex) } || + # Aws::Partitions.find { |p| p.name == 'aws' } + # + # return nil unless partition + # + # partition.metadata end # aws.parseArn(value: string) Option diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb index ede2b2d8b6c..4b7065d7663 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb @@ -20,7 +20,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin * `ENV['AWS_DEFAULT_REGION']` * `~/.aws/credentials` * `~/.aws/config` - DOCS + DOCS resolve_region(cfg) end @@ -35,7 +35,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin * `Aws.config[:sigv4a_signing_region_set]` * `ENV['AWS_SIGV4A_SIGNING_REGION_SET']` * `~/.aws/config` - DOCS + DOCS resolve_sigv4a_signing_region_set(cfg) end @@ -44,7 +44,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin docstring: <<-DOCS) do |cfg| When set to `true`, dualstack enabled endpoints (with `.aws` TLD) will be used if available. - DOCS + DOCS resolve_use_dualstack_endpoint(cfg) end @@ -54,7 +54,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin When set to `true`, fips compatible endpoints will be used if available. When a `fips` region is used, the region is normalized and this config is set to `true`. - DOCS + DOCS resolve_use_fips_endpoint(cfg) end @@ -67,7 +67,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin docstring: <<-DOCS) do |cfg| Setting to true disables use of endpoint URLs provided via environment variables and the shared configuration file. - DOCS + DOCS resolve_ignore_configured_endpoint_urls(cfg) end @@ -75,7 +75,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin The client endpoint is normally constructed from the `:region` option. You should only configure an `:endpoint` when connecting to test or custom endpoints. This should be a valid HTTP(S) URI. - DOCS + DOCS resolve_endpoint(cfg) end @@ -150,7 +150,18 @@ def resolve_endpoint(cfg) # that a custom endpoint has NOT been configured by the user cfg.override_config(:regional_endpoint, true) - resolve_legacy_endpoint(cfg) + # preserve legacy (pre EP2) client.config.endpoint still resolves a value + # when needed. + struct = cfg.instance_variable_get(:@struct) + if struct + # need a separate proc to get namespace/private access to resolve_legacy_endpoint + b = proc { resolve_legacy_endpoint(struct) } + struct.define_singleton_method(:endpoint) { @endpoint ||= b.call } + nil + else + # backup in case internal details of config resolver change + resolve_legacy_endpoint(cfg) + end end # get a custom configured endpoint from ENV or configuration @@ -213,7 +224,7 @@ def resolve_legacy_endpoint(cfg) sts_regional = cfg.sts_regional_endpoints end - Aws::Partitions::EndpointProvider.resolve( + endpoint = Aws::Partitions::EndpointProvider.resolve( cfg.region, endpoint_prefix, sts_regional, @@ -222,6 +233,7 @@ def resolve_legacy_endpoint(cfg) fips: cfg.use_fips_endpoint } ) + URI(endpoint) end end end diff --git a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb index c0bfe44e4f5..2ed7ecb852a 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb @@ -24,11 +24,8 @@ def add_handlers(handlers, config) end def after_initialize(client) - endpoint = client.config.endpoint - if endpoint.nil? - msg = "missing required option `:endpoint'" - raise ArgumentError, msg - end + # validate endpoint only if user set a custom endpoint + return if client.config.regional_endpoint endpoint = URI.parse(endpoint.to_s) if URI::HTTPS === endpoint or URI::HTTP === endpoint @@ -42,7 +39,9 @@ def after_initialize(client) class Handler < Client::Handler def call(context) - context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) + unless context.config.regional_endpoint + context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) + end @handler.call(context) end From 32a3e8ee95c79789763176f3685748f9b54c21f4 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 7 Oct 2024 13:38:36 -0700 Subject: [PATCH 02/18] fix whitespace --- gems/aws-partitions/lib/aws-partitions/partition_list.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/gems/aws-partitions/lib/aws-partitions/partition_list.rb b/gems/aws-partitions/lib/aws-partitions/partition_list.rb index 2fa56d158e0..002d61751c4 100644 --- a/gems/aws-partitions/lib/aws-partitions/partition_list.rb +++ b/gems/aws-partitions/lib/aws-partitions/partition_list.rb @@ -90,7 +90,6 @@ def build_metadata_regions(partition_name, metadata_regions, existing = nil) else regions[region_name] = Region.new( name: region_name, - description: region['description'], partition_name: partition_name, services: Set.new From 167dc31c1ebda4c2a6f5d96bbf68bad03a481041 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 7 Oct 2024 13:45:36 -0700 Subject: [PATCH 03/18] Fix issue in new aws_partition method --- gems/aws-partitions/lib/aws-partitions/metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gems/aws-partitions/lib/aws-partitions/metadata.rb b/gems/aws-partitions/lib/aws-partitions/metadata.rb index 60021dfc12a..eec329ae90e 100644 --- a/gems/aws-partitions/lib/aws-partitions/metadata.rb +++ b/gems/aws-partitions/lib/aws-partitions/metadata.rb @@ -7,7 +7,7 @@ module Metadata class << self def aws_partition(value) partition = - partitions.find { |p| p['regions']&.fetch(value) } || + partitions.find { |p| p['regions']&.fetch(value, nil) } || partitions.find { |p| value.match(p['regionRegex']) } || partitions.find { |p| p['id'] == 'aws' } From 3b43d7cbae6cbd358b4c04496facf66c561b6a76 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 7 Oct 2024 14:25:13 -0700 Subject: [PATCH 04/18] Fix failures --- gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb index 2ed7ecb852a..26b2118c4ea 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb @@ -25,8 +25,9 @@ def add_handlers(handlers, config) def after_initialize(client) # validate endpoint only if user set a custom endpoint - return if client.config.regional_endpoint + return if client.config.respond_to?(:regional_endpoint) && client.config.regional_endpoint + endpoint = client.config.endpoint endpoint = URI.parse(endpoint.to_s) if URI::HTTPS === endpoint or URI::HTTP === endpoint client.config.endpoint = endpoint @@ -39,7 +40,9 @@ def after_initialize(client) class Handler < Client::Handler def call(context) - unless context.config.regional_endpoint + if !context.config.respond_to?(:regional_endpoint) || # generic + !context.config.regional_endpoint || # custom endpoint + !context.config.respond_to?(:endpoint_provider) # legacy/pre-ep2/no rules context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) end @handler.call(context) From 25b35fc8f871ca5e0d072bbd542ef165f568f64a Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Tue, 8 Oct 2024 15:59:28 -0700 Subject: [PATCH 05/18] Update partition metadata method to match v4 --- gems/aws-partitions/lib/aws-partitions/metadata.rb | 10 ++++++---- .../lib/aws-sdk-core/endpoints/matchers.rb | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/gems/aws-partitions/lib/aws-partitions/metadata.rb b/gems/aws-partitions/lib/aws-partitions/metadata.rb index eec329ae90e..ee2ac0061f4 100644 --- a/gems/aws-partitions/lib/aws-partitions/metadata.rb +++ b/gems/aws-partitions/lib/aws-partitions/metadata.rb @@ -5,11 +5,13 @@ module Partitions # @api private module Metadata class << self - def aws_partition(value) + + # aws.partition(region: string) Option + def partition(region) partition = - partitions.find { |p| p['regions']&.fetch(value, nil) } || - partitions.find { |p| value.match(p['regionRegex']) } || - partitions.find { |p| p['id'] == 'aws' } + partitions.find { |p| p['regions']&.fetch(region, nil) } || + partitions.find { |p| region.match(p['regionRegex']) } || + partitions.find { |p| p['id'] == 'aws' } return nil unless partition diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb b/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb index 020070f4c1e..5d944cb1045 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb @@ -94,7 +94,7 @@ def self.valid_host_label?(value, allow_sub_domains = false) # aws.partition(value: string) Option def self.aws_partition(value) - Aws::Partitions::Metadata.aws_partition(value) + Aws::Partitions::Metadata.partition(value) # partition = # Aws::Partitions.find { |p| p.region?(value) } || # Aws::Partitions.find { |p| value.match(p.region_regex) } || From b9a04ed59ce938641306f3eb20af949ec987c737 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Tue, 8 Oct 2024 16:01:37 -0700 Subject: [PATCH 06/18] Remove commented out code --- gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb b/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb index 5d944cb1045..95ac2b17903 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb @@ -95,14 +95,6 @@ def self.valid_host_label?(value, allow_sub_domains = false) # aws.partition(value: string) Option def self.aws_partition(value) Aws::Partitions::Metadata.partition(value) - # partition = - # Aws::Partitions.find { |p| p.region?(value) } || - # Aws::Partitions.find { |p| value.match(p.region_regex) } || - # Aws::Partitions.find { |p| p.name == 'aws' } - # - # return nil unless partition - # - # partition.metadata end # aws.parseArn(value: string) Option From 13f2c57f5a89290d534d7386ecfa5e7128a7a047 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 9 Oct 2024 09:49:42 -0700 Subject: [PATCH 07/18] DEMO ALTERNATIVE: use cfg.endpoint_resolver to resolve config.endpoint --- .../aws-sdk-core/plugins/regional_endpoint.rb | 33 ++++++++++++++----- .../lib/seahorse/client/plugins/endpoint.rb | 14 ++++---- .../lib/aws-sdk-s3/endpoint_provider.rb | 3 ++ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb index 4b7065d7663..a24ae6f025d 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb @@ -79,6 +79,17 @@ class RegionalEndpoint < Seahorse::Client::Plugin resolve_endpoint(cfg) end + DefaultEndpointParameters = Struct.new( + :region, + :use_dual_stack, + :use_fips, + :endpoint + ) do + def method_missing(method_name, *_args) + nil + end + end + def after_initialize(client) region = client.config.region raise Errors::MissingRegionError if region.nil? || region == '' @@ -150,16 +161,20 @@ def resolve_endpoint(cfg) # that a custom endpoint has NOT been configured by the user cfg.override_config(:regional_endpoint, true) - # preserve legacy (pre EP2) client.config.endpoint still resolves a value - # when needed. - struct = cfg.instance_variable_get(:@struct) - if struct - # need a separate proc to get namespace/private access to resolve_legacy_endpoint - b = proc { resolve_legacy_endpoint(struct) } - struct.define_singleton_method(:endpoint) { @endpoint ||= b.call } - nil + # preserve legacy (pre EP2) client.config.endpoint + if cfg.respond_to?(:endpoint_provider) && (endpoint_provider = cfg.endpoint_provider) + params = DefaultEndpointParameters.new + params.region = cfg.region + params.use_dual_stack = cfg.use_dualstack_endpoint + params.use_fips = cfg.use_fips_endpoint + begin + endpoint = endpoint_provider.resolve_endpoint(params) + endpoint.url + rescue ArgumentError + # fallback to legacy + resolve_legacy_endpoint(cfg) + end else - # backup in case internal details of config resolver change resolve_legacy_endpoint(cfg) end end diff --git a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb index 26b2118c4ea..c0bfe44e4f5 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb @@ -24,10 +24,12 @@ def add_handlers(handlers, config) end def after_initialize(client) - # validate endpoint only if user set a custom endpoint - return if client.config.respond_to?(:regional_endpoint) && client.config.regional_endpoint - endpoint = client.config.endpoint + if endpoint.nil? + msg = "missing required option `:endpoint'" + raise ArgumentError, msg + end + endpoint = URI.parse(endpoint.to_s) if URI::HTTPS === endpoint or URI::HTTP === endpoint client.config.endpoint = endpoint @@ -40,11 +42,7 @@ def after_initialize(client) class Handler < Client::Handler def call(context) - if !context.config.respond_to?(:regional_endpoint) || # generic - !context.config.regional_endpoint || # custom endpoint - !context.config.respond_to?(:endpoint_provider) # legacy/pre-ep2/no rules - context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) - end + context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) @handler.call(context) end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/endpoint_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/endpoint_provider.rb index ff0e5c54356..4a64432ffc0 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/endpoint_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/endpoint_provider.rb @@ -27,6 +27,9 @@ def resolve_endpoint(parameters) use_arn_region = parameters.use_arn_region use_s3_express_control_endpoint = parameters.use_s3_express_control_endpoint disable_s3_express_session_auth = parameters.disable_s3_express_session_auth + require 'byebug' + byebug + if Aws::Endpoints::Matchers.set?(region) if Aws::Endpoints::Matchers.boolean_equals?(accelerate, true) && Aws::Endpoints::Matchers.boolean_equals?(use_fips, true) raise ArgumentError, "Accelerate cannot be used with FIPS" From b539c55dcba270c4069197969ca9ac1d418f3374 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 9 Oct 2024 09:51:38 -0700 Subject: [PATCH 08/18] Revert "DEMO ALTERNATIVE: use cfg.endpoint_resolver to resolve config.endpoint" This reverts commit 13f2c57f5a89290d534d7386ecfa5e7128a7a047. --- .../aws-sdk-core/plugins/regional_endpoint.rb | 33 +++++-------------- .../lib/seahorse/client/plugins/endpoint.rb | 14 ++++---- .../lib/aws-sdk-s3/endpoint_provider.rb | 3 -- 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb index a24ae6f025d..4b7065d7663 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb @@ -79,17 +79,6 @@ class RegionalEndpoint < Seahorse::Client::Plugin resolve_endpoint(cfg) end - DefaultEndpointParameters = Struct.new( - :region, - :use_dual_stack, - :use_fips, - :endpoint - ) do - def method_missing(method_name, *_args) - nil - end - end - def after_initialize(client) region = client.config.region raise Errors::MissingRegionError if region.nil? || region == '' @@ -161,20 +150,16 @@ def resolve_endpoint(cfg) # that a custom endpoint has NOT been configured by the user cfg.override_config(:regional_endpoint, true) - # preserve legacy (pre EP2) client.config.endpoint - if cfg.respond_to?(:endpoint_provider) && (endpoint_provider = cfg.endpoint_provider) - params = DefaultEndpointParameters.new - params.region = cfg.region - params.use_dual_stack = cfg.use_dualstack_endpoint - params.use_fips = cfg.use_fips_endpoint - begin - endpoint = endpoint_provider.resolve_endpoint(params) - endpoint.url - rescue ArgumentError - # fallback to legacy - resolve_legacy_endpoint(cfg) - end + # preserve legacy (pre EP2) client.config.endpoint still resolves a value + # when needed. + struct = cfg.instance_variable_get(:@struct) + if struct + # need a separate proc to get namespace/private access to resolve_legacy_endpoint + b = proc { resolve_legacy_endpoint(struct) } + struct.define_singleton_method(:endpoint) { @endpoint ||= b.call } + nil else + # backup in case internal details of config resolver change resolve_legacy_endpoint(cfg) end end diff --git a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb index c0bfe44e4f5..26b2118c4ea 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb @@ -24,12 +24,10 @@ def add_handlers(handlers, config) end def after_initialize(client) - endpoint = client.config.endpoint - if endpoint.nil? - msg = "missing required option `:endpoint'" - raise ArgumentError, msg - end + # validate endpoint only if user set a custom endpoint + return if client.config.respond_to?(:regional_endpoint) && client.config.regional_endpoint + endpoint = client.config.endpoint endpoint = URI.parse(endpoint.to_s) if URI::HTTPS === endpoint or URI::HTTP === endpoint client.config.endpoint = endpoint @@ -42,7 +40,11 @@ def after_initialize(client) class Handler < Client::Handler def call(context) - context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) + if !context.config.respond_to?(:regional_endpoint) || # generic + !context.config.regional_endpoint || # custom endpoint + !context.config.respond_to?(:endpoint_provider) # legacy/pre-ep2/no rules + context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) + end @handler.call(context) end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/endpoint_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/endpoint_provider.rb index 4a64432ffc0..ff0e5c54356 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/endpoint_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/endpoint_provider.rb @@ -27,9 +27,6 @@ def resolve_endpoint(parameters) use_arn_region = parameters.use_arn_region use_s3_express_control_endpoint = parameters.use_s3_express_control_endpoint disable_s3_express_session_auth = parameters.disable_s3_express_session_auth - require 'byebug' - byebug - if Aws::Endpoints::Matchers.set?(region) if Aws::Endpoints::Matchers.boolean_equals?(accelerate, true) && Aws::Endpoints::Matchers.boolean_equals?(use_fips, true) raise ArgumentError, "Accelerate cannot be used with FIPS" From 61e8de1829a3c5c6027366ddf895191a04219ffa Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 9 Oct 2024 10:08:30 -0700 Subject: [PATCH 09/18] Update min partition version --- gems/aws-sdk-core/aws-sdk-core.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gems/aws-sdk-core/aws-sdk-core.gemspec b/gems/aws-sdk-core/aws-sdk-core.gemspec index 402b98a10a8..95bb2006e4e 100644 --- a/gems/aws-sdk-core/aws-sdk-core.gemspec +++ b/gems/aws-sdk-core/aws-sdk-core.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |spec| spec.files = Dir['LICENSE.txt', 'CHANGELOG.md', 'VERSION', 'lib/**/*.rb', 'sig/**/*.rbs', 'ca-bundle.crt'] spec.add_dependency('jmespath', '~> 1', '>= 1.6.1') # necessary for secure jmespath JSON parsing - spec.add_dependency('aws-partitions', '~> 1', '>= 1.987.0') # necessary for new endpoint resolution + spec.add_dependency('aws-partitions', '~> 1', '>= 1.988.0') # necessary for new endpoint resolution spec.add_dependency('aws-sigv4', '~> 1.9') # necessary for s3 express auth/native sigv4a support spec.add_dependency('aws-eventstream', '~> 1', '>= 1.3.0') # necessary for binary eventstream From 5516a5af992eec8a2513efdb4f478a420aef1e11 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 9 Oct 2024 11:54:12 -0700 Subject: [PATCH 10/18] Add struct attr_reader on DefaultResolver --- .../lib/aws-sdk-core/plugins/regional_endpoint.rb | 14 ++++---------- .../lib/seahorse/client/configuration.rb | 2 ++ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb index 4b7065d7663..ffffc17088b 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb @@ -152,16 +152,10 @@ def resolve_endpoint(cfg) # preserve legacy (pre EP2) client.config.endpoint still resolves a value # when needed. - struct = cfg.instance_variable_get(:@struct) - if struct - # need a separate proc to get namespace/private access to resolve_legacy_endpoint - b = proc { resolve_legacy_endpoint(struct) } - struct.define_singleton_method(:endpoint) { @endpoint ||= b.call } - nil - else - # backup in case internal details of config resolver change - resolve_legacy_endpoint(cfg) - end + # need a separate proc to get namespace/private access to resolve_legacy_endpoint + b = proc { resolve_legacy_endpoint(cfg.struct) } + cfg.struct.define_singleton_method(:endpoint) { @endpoint ||= b.call } + nil end # get a custom configured endpoint from ENV or configuration diff --git a/gems/aws-sdk-core/lib/seahorse/client/configuration.rb b/gems/aws-sdk-core/lib/seahorse/client/configuration.rb index 683a053d26d..693214e63e4 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/configuration.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/configuration.rb @@ -187,6 +187,8 @@ def initialize(struct) @members = Set.new(@struct.members) end + attr_reader :struct + def resolve @members.each { |opt_name| value_at(opt_name) } end From 3074fcec2e18ece291f48904f744ece7a638b0d9 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 9 Oct 2024 12:06:56 -0700 Subject: [PATCH 11/18] Remove endpoint plugin from ep2.0 services --- .../aws-sdk-code-generator/views/client_class.rb | 10 ++++++++++ .../templates/client_class.mustache | 4 ++++ .../lib/seahorse/client/plugins/endpoint.rb | 14 ++++++-------- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_class.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_class.rb index 43129758665..18272165177 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_class.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_class.rb @@ -19,6 +19,7 @@ class ClientClass < View # @option options [Hash] :waiters # @option options [Hash] :client_examples # @option options [Array] + def remove_plugin_class_names + # these differ from PluginList's set of remove_plugins + # these apply to default plugins in Seahorse::Client::Base + unless @legacy_endpoints + ['Seahorse::Client::Plugins::Endpoint'] + end + end # @return [Array] def waiters @waiters diff --git a/build_tools/aws-sdk-code-generator/templates/client_class.mustache b/build_tools/aws-sdk-code-generator/templates/client_class.mustache index 999190344d8..05a48ad79b9 100644 --- a/build_tools/aws-sdk-code-generator/templates/client_class.mustache +++ b/build_tools/aws-sdk-code-generator/templates/client_class.mustache @@ -32,6 +32,10 @@ module {{module_name}} add_plugin({{{.}}}) {{/plugin_class_names}} + {{#remove_plugin_class_names}} + remove_plugin({{{.}}}) + {{/remove_plugin_class_names}} + {{#client_constructor}} # @overload initialize(options) # @param [Hash] options diff --git a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb index 26b2118c4ea..c0bfe44e4f5 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb @@ -24,10 +24,12 @@ def add_handlers(handlers, config) end def after_initialize(client) - # validate endpoint only if user set a custom endpoint - return if client.config.respond_to?(:regional_endpoint) && client.config.regional_endpoint - endpoint = client.config.endpoint + if endpoint.nil? + msg = "missing required option `:endpoint'" + raise ArgumentError, msg + end + endpoint = URI.parse(endpoint.to_s) if URI::HTTPS === endpoint or URI::HTTP === endpoint client.config.endpoint = endpoint @@ -40,11 +42,7 @@ def after_initialize(client) class Handler < Client::Handler def call(context) - if !context.config.respond_to?(:regional_endpoint) || # generic - !context.config.regional_endpoint || # custom endpoint - !context.config.respond_to?(:endpoint_provider) # legacy/pre-ep2/no rules - context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) - end + context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) @handler.call(context) end From f8b9164620ef0f4344b26e333b3aac459e63b17a Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 9 Oct 2024 14:22:30 -0700 Subject: [PATCH 12/18] Revert "Remove endpoint plugin from ep2.0 services" This reverts commit 3074fcec2e18ece291f48904f744ece7a638b0d9. --- .../aws-sdk-code-generator/views/client_class.rb | 10 ---------- .../templates/client_class.mustache | 4 ---- .../lib/seahorse/client/plugins/endpoint.rb | 14 ++++++++------ 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_class.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_class.rb index 18272165177..43129758665 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_class.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_class.rb @@ -19,7 +19,6 @@ class ClientClass < View # @option options [Hash] :waiters # @option options [Hash] :client_examples # @option options [Array] - def remove_plugin_class_names - # these differ from PluginList's set of remove_plugins - # these apply to default plugins in Seahorse::Client::Base - unless @legacy_endpoints - ['Seahorse::Client::Plugins::Endpoint'] - end - end # @return [Array] def waiters @waiters diff --git a/build_tools/aws-sdk-code-generator/templates/client_class.mustache b/build_tools/aws-sdk-code-generator/templates/client_class.mustache index 05a48ad79b9..999190344d8 100644 --- a/build_tools/aws-sdk-code-generator/templates/client_class.mustache +++ b/build_tools/aws-sdk-code-generator/templates/client_class.mustache @@ -32,10 +32,6 @@ module {{module_name}} add_plugin({{{.}}}) {{/plugin_class_names}} - {{#remove_plugin_class_names}} - remove_plugin({{{.}}}) - {{/remove_plugin_class_names}} - {{#client_constructor}} # @overload initialize(options) # @param [Hash] options diff --git a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb index c0bfe44e4f5..26b2118c4ea 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb @@ -24,12 +24,10 @@ def add_handlers(handlers, config) end def after_initialize(client) - endpoint = client.config.endpoint - if endpoint.nil? - msg = "missing required option `:endpoint'" - raise ArgumentError, msg - end + # validate endpoint only if user set a custom endpoint + return if client.config.respond_to?(:regional_endpoint) && client.config.regional_endpoint + endpoint = client.config.endpoint endpoint = URI.parse(endpoint.to_s) if URI::HTTPS === endpoint or URI::HTTP === endpoint client.config.endpoint = endpoint @@ -42,7 +40,11 @@ def after_initialize(client) class Handler < Client::Handler def call(context) - context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) + if !context.config.respond_to?(:regional_endpoint) || # generic + !context.config.regional_endpoint || # custom endpoint + !context.config.respond_to?(:endpoint_provider) # legacy/pre-ep2/no rules + context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) + end @handler.call(context) end From 3ea1bafb990d44a596a7ad666cba69942da35ca5 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 9 Oct 2024 14:33:09 -0700 Subject: [PATCH 13/18] New approach - use the EndpointProvider (ep2) to resolve a default endpoint in after_initialize --- .../aws-sdk-core/plugins/regional_endpoint.rb | 88 ++++++++++++++----- .../lib/seahorse/client/configuration.rb | 2 - gems/aws-sdk-core/spec/aws/endpoints_spec.rb | 2 +- 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb index ffffc17088b..a0b64463658 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb @@ -83,6 +83,9 @@ def after_initialize(client) region = client.config.region raise Errors::MissingRegionError if region.nil? || region == '' + # resolve a default endpoint to preserve legacy behavior + initialize_default_endpoint(client) if client.config.endpoint.nil? + region_set = client.config.sigv4a_signing_region_set return if region_set.nil? raise Errors::InvalidRegionSetError unless region_set.is_a?(Array) @@ -93,6 +96,66 @@ def after_initialize(client) client.config.sigv4a_signing_region_set = region_set end + private + + def initialize_default_endpoint(client) + client_module = Object.const_get(client.class.name.rpartition('::').first) + if client.config.respond_to?(:endpoint_provider) && + client_module.const_defined?(:EndpointParameters) && + (param_class = client_module.const_get(:EndpointParameters)) && + (endpoint_provider = client.config.endpoint_provider) + + params = build_parameters(param_class.new, client.config) + begin + endpoint = endpoint_provider.resolve_endpoint(params) + client.config.endpoint = endpoint.url + rescue ArgumentError + # fallback to legacy + client.config.endpoint = resolve_legacy_endpoint(client.config) + end + else + # fallback to legacy + client.config.endpoint = resolve_legacy_endpoint(client.config) + end + end + + def build_parameters(params, config) + # generic parameters set for most (but not all) services + params.region = config.region if params.respond_to?(:region=) + params.use_dual_stack = config.use_dualstack_endpoint if params.respond_to?(:use_dual_stack=) + params.use_fips = config.use_fips_endpoint if params.respond_to?(:use_fips=) + + # service specific + if config.respond_to?(:sts_regional_endpoints) && params.respond_to?(:use_global_endpoint=) + params.use_global_endpoint = (config.sts_regional_endpoints == 'legacy') + end + + if config.respond_to?(:s3_us_east_1_regional_endpoint) && params.respond_to?(:use_global_endpoint=) + params.use_global_endpoint = (config.s3_us_east_1_regional_endpoint == 'legacy') + end + + params + end + + # set a default endpoint in config using legacy (endpoints.json) resolver + def resolve_legacy_endpoint(cfg) + endpoint_prefix = cfg.api.metadata['endpointPrefix'] + if cfg.respond_to?(:sts_regional_endpoints) + sts_regional = cfg.sts_regional_endpoints + end + + endpoint = Aws::Partitions::EndpointProvider.resolve( + cfg.region, + endpoint_prefix, + sts_regional, + { + dualstack: cfg.use_dualstack_endpoint, + fips: cfg.use_fips_endpoint + } + ) + URI(endpoint) + end + class << self private @@ -150,11 +213,7 @@ def resolve_endpoint(cfg) # that a custom endpoint has NOT been configured by the user cfg.override_config(:regional_endpoint, true) - # preserve legacy (pre EP2) client.config.endpoint still resolves a value - # when needed. - # need a separate proc to get namespace/private access to resolve_legacy_endpoint - b = proc { resolve_legacy_endpoint(cfg.struct) } - cfg.struct.define_singleton_method(:endpoint) { @endpoint ||= b.call } + # a default endpoint is resolved in after_initialize nil end @@ -210,25 +269,6 @@ def handle_legacy_pseudo_regions(cfg) cfg.override_config(:region, new_region) end end - - # set a default endpoint in config using legacy (endpoints.json) resolver - def resolve_legacy_endpoint(cfg) - endpoint_prefix = cfg.api.metadata['endpointPrefix'] - if cfg.respond_to?(:sts_regional_endpoints) - sts_regional = cfg.sts_regional_endpoints - end - - endpoint = Aws::Partitions::EndpointProvider.resolve( - cfg.region, - endpoint_prefix, - sts_regional, - { - dualstack: cfg.use_dualstack_endpoint, - fips: cfg.use_fips_endpoint - } - ) - URI(endpoint) - end end end end diff --git a/gems/aws-sdk-core/lib/seahorse/client/configuration.rb b/gems/aws-sdk-core/lib/seahorse/client/configuration.rb index 693214e63e4..683a053d26d 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/configuration.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/configuration.rb @@ -187,8 +187,6 @@ def initialize(struct) @members = Set.new(@struct.members) end - attr_reader :struct - def resolve @members.each { |opt_name| value_at(opt_name) } end diff --git a/gems/aws-sdk-core/spec/aws/endpoints_spec.rb b/gems/aws-sdk-core/spec/aws/endpoints_spec.rb index 87f6bb106f2..78e84edebe9 100644 --- a/gems/aws-sdk-core/spec/aws/endpoints_spec.rb +++ b/gems/aws-sdk-core/spec/aws/endpoints_spec.rb @@ -66,7 +66,7 @@ module Aws before do expect_any_instance_of(endpoints_service.const_get(:EndpointProvider)) - .to receive(:resolve_endpoint).and_return(endpoint) + .to receive(:resolve_endpoint).at_least(:once).and_return(endpoint) end describe '#resolve_auth_scheme' do From d47a8c52b1f70d8b417ba5b16456dcbf81e9823a Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Thu, 10 Oct 2024 11:29:29 -0700 Subject: [PATCH 14/18] Revert changes to searhose endpoint plugin --- .../lib/seahorse/client/plugins/endpoint.rb | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb index 26b2118c4ea..c0bfe44e4f5 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb @@ -24,10 +24,12 @@ def add_handlers(handlers, config) end def after_initialize(client) - # validate endpoint only if user set a custom endpoint - return if client.config.respond_to?(:regional_endpoint) && client.config.regional_endpoint - endpoint = client.config.endpoint + if endpoint.nil? + msg = "missing required option `:endpoint'" + raise ArgumentError, msg + end + endpoint = URI.parse(endpoint.to_s) if URI::HTTPS === endpoint or URI::HTTP === endpoint client.config.endpoint = endpoint @@ -40,11 +42,7 @@ def after_initialize(client) class Handler < Client::Handler def call(context) - if !context.config.respond_to?(:regional_endpoint) || # generic - !context.config.regional_endpoint || # custom endpoint - !context.config.respond_to?(:endpoint_provider) # legacy/pre-ep2/no rules - context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) - end + context.http_request.endpoint = URI.parse(context.config.endpoint.to_s) @handler.call(context) end From a17e8705e3ddcae8974101addb433798f6e4295f Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Thu, 10 Oct 2024 11:53:32 -0700 Subject: [PATCH 15/18] Fix tests --- .../spec/interfaces/plugins/endpoints_spec.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb b/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb index 71c815a7761..2d9fdc08ac2 100644 --- a/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb +++ b/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb @@ -38,6 +38,7 @@ it 'uses endpoint provider with params to resolve endpoint' do expect_any_instance_of(EndpointsPlugin::EndpointProvider) .to receive(:resolve_endpoint) + .at_least(:once) .with(an_instance_of(EndpointsPlugin::EndpointParameters)) .and_call_original client.operation @@ -124,6 +125,10 @@ EndpointsPrecedence::Client.new(stub_responses: true, region: 'config') end + before(:each) do + client # force client initialization before testing resolution + end + # Most to least # staticContextParams # contextParam From f3295502ad1c0b770abf7ac490b63939d335ac0d Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Tue, 15 Oct 2024 15:00:17 -0700 Subject: [PATCH 16/18] New approach, change endpoint generation and add create class method --- .../lib/aws-sdk-code-generator.rb | 1 + .../endpoint_parameter.rb | 200 ++++++++++++++++++ .../views/endpoint_parameters_class.rb | 60 +----- .../views/endpoints_module.rb | 156 ++------------ .../spec/interfaces/plugins/endpoints_spec.rb | 6 +- .../endpoint_parameters_class.mustache | 24 ++- .../templates/endpoints_module.mustache | 14 +- .../templates/endpoints_plugin.mustache | 8 +- .../aws-sdk-core/plugins/regional_endpoint.rb | 21 +- 9 files changed, 256 insertions(+), 234 deletions(-) create mode 100644 build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/endpoint_parameter.rb diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator.rb index 09fad189182..23fc74b92f2 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator.rb @@ -12,6 +12,7 @@ require_relative 'aws-sdk-code-generator/client_response_structure_example' require_relative 'aws-sdk-code-generator/crosslink' require_relative 'aws-sdk-code-generator/docstring' +require_relative 'aws-sdk-code-generator/endpoint_parameter' require_relative 'aws-sdk-code-generator/hash_formatter' require_relative 'aws-sdk-code-generator/helper' require_relative 'aws-sdk-code-generator/plugin_list' diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/endpoint_parameter.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/endpoint_parameter.rb new file mode 100644 index 00000000000..f84637660ab --- /dev/null +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/endpoint_parameter.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +module AwsSdkCodeGenerator + class EndpointParameter + def initialize(name, definition, value, source) + @name = name + @source = source + @value = value + + @type = definition['type'] + @built_in = definition['builtIn'] + @default = definition['default'] + @required = definition['required'] + @documentation = "# @!attribute #{key}\n" + if definition['documentation'] + @documentation += " # #{definition['documentation']}\n" + end + if deprecated = definition['deprecated'] + @documentation += " #\n # @deprecated\n" + if deprecated['message'] + @documentation += " # #{deprecated['message']}\n" + end + if deprecated['since'] + @documentation += " # Since: #{deprecated['since']}\n" + end + end + @documentation += " #\n # @return [#{@type}]\n #" + end + + # @return [String] + attr_reader :name + + # @return [String] + attr_reader :documentation + + # @return [Boolean] + attr_reader :required + + # @return [String] + attr_reader :source + + # @return [String] + attr_reader :value + + # @return [String,Boolean,Array] + def default + case @default + when String + "\"#{@default}\"" + else + @default.to_s + end + end + + def default? + !@default.nil? + end + + def validate_required? + required && !default? + end + def key + Underscore.underscore(name) + end + + class << self + + def operation_specific_params?(service, operation) + parameters = service.endpoint_rules.fetch('parameters', {}) + parameters.each do |param_name, param_data| + _value, source = endpoint_parameter_value(service, param_name, param_data, operation) + return true if source == 'operation' + end + false + end + + # Most to least + # staticContextParams + # contextParam + # operationContextParams + # clientContextParams (always sourced from config) + # Built-In Bindings (sourced from config in most cases, context in some cases to allow operation level overrides) + # Built-in binding default values + # @retyrn [value, source]. source may be one of [operation, config, default] + def endpoint_parameter_value(service, param_name, param_data, operation=nil) + unless operation.nil? + value, source = [ + static_context_param(operation, param_name), 'operation' + ] + value, source = [ + context_param_value(service, operation, param_name), 'operation' + ] unless value + value, source = [ + operation_context_param_value(operation, param_name), 'operation' + ] unless value + end + + value, source = [ + client_context_param_value(service, param_name, param_data), + 'config' + ] unless value + + + # built-ins may be sourced from operation context in some cases + unless value + value, source = built_in_param_value(service, param_data) + end + + [value || 'nil', source || 'default'] + end + + def client_context_param_value(service, param_name, param_data) + if service.api['clientContextParams']&.key?(param_name) && + !param_data['builtIn'] + "config.#{Underscore.underscore(param_name)}" + end + end + + # built-ins may be sourced from operation context in some cases + def built_in_param_value(service, param_data) + source = 'config' + value = + case param_data['builtIn'] + when 'AWS::Region' + 'config.region' + when 'AWS::UseFIPS' + 'config.use_fips_endpoint' + when 'AWS::UseDualStack' + if service.name == 'S3' || service.name == 'S3Control' + source = 'operation' + 'context[:use_dualstack_endpoint]' + else + 'config.use_dualstack_endpoint' + end + when 'AWS::Auth::AccountId' + 'config.credentials.credentials.account_id' + when 'AWS::Auth::AccountIdEndpointMode' + 'config.account_id_endpoint_mode' + when 'AWS::STS::UseGlobalEndpoint' + "config.sts_regional_endpoints == 'legacy'" + when 'AWS::S3::UseGlobalEndpoint' + "config.s3_us_east_1_regional_endpoint == 'legacy'" + when 'AWS::S3::Accelerate' + if service.name == 'S3' || service.name == 'S3Control' + source = 'operation' + 'context[:use_accelerate_endpoint]' + else + 'config.use_accelerate_endpoint' + end + when 'AWS::S3::ForcePathStyle' + 'config.force_path_style' + when 'AWS::S3::UseArnRegion', 'AWS::S3Control::UseArnRegion' + 'config.s3_use_arn_region' + when 'AWS::S3::DisableMultiRegionAccessPoints' + 'config.s3_disable_multiregion_access_points' + when 'SDK::Endpoint' + 'config.regional_endpoint ? nil : config.endpoint.to_s' + else + source = nil + nil # no value, not a default + end + [value, source] + end + + def context_param_value(service, operation, param_name) + return nil unless operation['input'] + + input_shape = operation['input']['shape'] + members = service.api['shapes'][input_shape].fetch('members', {}) + members.detect do |(member_name, member)| + context_param = member.fetch('contextParam', {}) + if context_param.fetch('name', nil) == param_name + break "context.params[:#{Underscore.underscore(member_name)}]" + end + end + end + + def operation_context_param_value(operation, param_name) + return nil unless operation['input'] + + binding = operation.fetch('operationContextParams', {})[param_name] + + return nil unless binding + + "JMESPath.search(\"#{Underscore.underscore_jmespath(binding['path'])}\", context.params)" + end + + def static_context_param(operation, param_name) + value = operation.fetch('staticContextParams', {}) + .fetch(param_name, {}).fetch('value', nil) + if !value.nil? && value.is_a?(String) + "\"#{value}\"" + else + value + end + end + end + end + +end \ No newline at end of file diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoint_parameters_class.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoint_parameters_class.rb index e0afcbb2786..7c17a98dbff 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoint_parameters_class.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoint_parameters_class.rb @@ -9,7 +9,8 @@ def initialize(options) @service = options.fetch(:service) if (parameters = @service.endpoint_rules&.fetch('parameters')) @parameters = parameters.map do |k,p| - EndpointParameter.new(k, p) + value, source = EndpointParameter.endpoint_parameter_value(@service, k, p) + EndpointParameter.new(k, p, value, source) end end end @@ -17,6 +18,11 @@ def initialize(options) # @return [Array] attr_reader :parameters + # @return [Array] + def config_parameters + parameters.select { |p| p.source == 'config' } + end + # @return [String|nil] def generated_src_warning return if @service.protocol == 'api-gateway' @@ -26,58 +32,6 @@ def generated_src_warning def module_name @service.module_name end - - class EndpointParameter - def initialize(name, definition={}) - @name = name - @type = definition['type'] - @built_in = definition['builtIn'] - @default = definition['default'] - @required = definition['required'] - @documentation = "# @!attribute #{underscore_name}\n" - if definition['documentation'] - @documentation += " # #{definition['documentation']}\n" - end - if deprecated = definition['deprecated'] - @documentation += " #\n # @deprecated\n" - if deprecated['message'] - @documentation += " # #{deprecated['message']}\n" - end - if deprecated['since'] - @documentation += " # Since: #{deprecated['since']}\n" - end - end - @documentation += " #\n # @return [#{@type}]\n #" - end - - # @return [String] - attr_reader :name - - # @return [String] - attr_reader :documentation - - # @return [Boolean] - attr_reader :required - - # @return [String,Boolean,Array] - def default - case @default - when String - "\"#{@default}\"" - else - @default.to_s - end - end - - def default? - !@default.nil? - end - - def underscore_name - Underscore.underscore(name) - end - end - end end end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoints_module.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoints_module.rb index 0953ad49595..a409635e43c 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoints_module.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoints_module.rb @@ -9,13 +9,13 @@ def initialize(options) @parameters = @service.endpoint_rules.fetch('parameters', {}) - @endpoint_classes = @service.api['operations'].each.with_object([]) do - |(name, op), array| - array << EndpointClass.new( + @endpoint_classes = @service.api['operations'].map do + |name, op| + EndpointClass.new( name: name, parameters: endpoint_parameters_for_operation(op) ) - end + end.reject { |ec| ec.parameters.empty? } end # @return [Array] @@ -34,157 +34,35 @@ def module_name class EndpointClass def initialize(options) @name = options[:name] + @operation = Underscore.underscore(@name) @parameters = options[:parameters] end # @return [String] attr_reader :name - # @return [Array] - attr_reader :parameters - end - - class EndpointParameter - def initialize(options) - @key = options[:key] - @value = options[:value] - @source = options[:source] - @param_data = options[:param_data] - end - - # @return [String] - attr_accessor :key - # @return [String] - attr_accessor :value + attr_reader :operation - # @return [String] - attr_accessor :source - - # @return [Hash] - attr_accessor :param_data - - def static_string? - @source == 'staticContextParam' && value.is_a?(String) - end + # @return [Array] + attr_reader :parameters end - private def endpoint_parameters_for_operation(operation) - @parameters.each.with_object([]) do |(param_name, param_data), endpoint_parameters| - value, source = endpoint_parameter_value( - operation, param_name, param_data + @parameters.map do |param_name, param_data| + value, source = EndpointParameter.endpoint_parameter_value( + @service, param_name, param_data, operation ) - endpoint_parameters << EndpointParameter.new( - key: Underscore.underscore(param_name), - value: value, - source: source, - param_data: param_data + EndpointParameter.new( + param_name, + param_data, + value, + source ) - end - end - - # Most to least - # staticContextParams - # contextParam - # operationContextParams - # clientContextParams - # Built-In Bindings - # Built-in binding default values - def endpoint_parameter_value(operation, param_name, param_data) - value, source = [ - static_context_param(operation, param_name), 'staticContextParam' - ] - value, source = [ - context_param_value(operation, param_name), 'contextParam' - ] unless value - value, source = [ - operation_context_param_value(operation, param_name), 'operationContextParam' - ] unless value - value, source = [ - client_context_param_value(param_name, param_data), - 'clientContextParam' - ] unless value - value, source = [ - built_in_client_context_param_value(param_data), 'builtIn' - ] unless value - - [value || 'nil', source] - end - - def client_context_param_value(param_name, param_data) - if @service.api['clientContextParams']&.key?(param_name) && - !param_data['builtIn'] - "context.config.#{Underscore.underscore(param_name)}" - end - end - - def built_in_client_context_param_value(param_data) - case param_data['builtIn'] - when 'AWS::Region' - 'context.config.region' - when 'AWS::UseFIPS' - 'context.config.use_fips_endpoint' - when 'AWS::UseDualStack' - if @service.name == 'S3' || @service.name == 'S3Control' - 'context[:use_dualstack_endpoint]' - else - 'context.config.use_dualstack_endpoint' - end - when 'AWS::Auth::AccountId' - 'context.config.credentials.credentials.account_id' - when 'AWS::Auth::AccountIdEndpointMode' - 'context.config.account_id_endpoint_mode' - when 'AWS::STS::UseGlobalEndpoint' - "context.config.sts_regional_endpoints == 'legacy'" - when 'AWS::S3::UseGlobalEndpoint' - "context.config.s3_us_east_1_regional_endpoint == 'legacy'" - when 'AWS::S3::Accelerate' - if @service.name == 'S3' || @service.name == 'S3Control' - 'context[:use_accelerate_endpoint]' - else - 'context.config.use_accelerate_endpoint' - end - when 'AWS::S3::ForcePathStyle' - 'context.config.force_path_style' - when 'AWS::S3::UseArnRegion', 'AWS::S3Control::UseArnRegion' - 'context.config.s3_use_arn_region' - when 'AWS::S3::DisableMultiRegionAccessPoints' - 'context.config.s3_disable_multiregion_access_points' - when 'SDK::Endpoint' - 'context.config.regional_endpoint ? nil : context.config.endpoint.to_s' - end - end - - def context_param_value(operation, param_name) - return nil unless operation['input'] - - input_shape = operation['input']['shape'] - members = @service.api['shapes'][input_shape].fetch('members', {}) - members.detect do |(member_name, member)| - context_param = member.fetch('contextParam', {}) - if context_param.fetch('name', nil) == param_name - break "context.params[:#{Underscore.underscore(member_name)}]" - end - end - end - - def operation_context_param_value(operation, param_name) - return nil unless operation['input'] - - binding = operation.fetch('operationContextParams', {})[param_name] - - return nil unless binding - - "JMESPath.search(\"#{Underscore.underscore_jmespath(binding['path'])}\", context.params)" - end - - def static_context_param(operation, param_name) - operation.fetch('staticContextParams', {}) - .fetch(param_name, {}).fetch('value', nil) + end.select { |p| p.source == 'operation' } end end end diff --git a/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb b/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb index 2d9fdc08ac2..4dab07d5282 100644 --- a/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb +++ b/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb @@ -72,13 +72,13 @@ it 'maps built ins' do expect(EndpointsBuiltIns::EndpointParameters).to receive(:new) - .with( + .with({ region: region, endpoint: endpoint, use_fips: true, use_dual_stack: true, account_id_endpoint_mode: 'preferred' - ).and_call_original + }).and_call_original client.operation end @@ -171,7 +171,7 @@ it 'defaults to nil' do client = EndpointsPrecedence::Client.new(stub_responses: true) expect(EndpointsPrecedence::EndpointParameters).to receive(:new) - .with(hash_including(nothing: nil)).and_call_original + .with(hash_including(client_context_param: nil)).and_call_original client.operation end end diff --git a/build_tools/aws-sdk-code-generator/templates/endpoint_parameters_class.mustache b/build_tools/aws-sdk-code-generator/templates/endpoint_parameters_class.mustache index 0b9b3278791..e921ad00164 100644 --- a/build_tools/aws-sdk-code-generator/templates/endpoint_parameters_class.mustache +++ b/build_tools/aws-sdk-code-generator/templates/endpoint_parameters_class.mustache @@ -11,7 +11,7 @@ module {{module_name}} {{/parameters}} EndpointParameters = Struct.new( {{#parameters}} - :{{underscore_name}}, + :{{key}}, {{/parameters}} {{^parameters}} nil @@ -23,23 +23,31 @@ module {{module_name}} class << self PARAM_MAP = { {{#parameters}} - '{{name}}' => :{{underscore_name}}, + '{{name}}' => :{{key}}, {{/parameters}} }.freeze end def initialize(options = {}) {{#parameters}} - self[:{{underscore_name}}] = options[:{{underscore_name}}] + self[:{{key}}] = options[:{{key}}] {{#default?}} - self[:{{underscore_name}}] = {{{default}}} if self[:{{underscore_name}}].nil? + self[:{{key}}] = {{{default}}} if self[:{{key}}].nil? {{/default?}} - {{#required}} - if self[:{{underscore_name}}].nil? - raise ArgumentError, "Missing required EndpointParameter: :{{underscore_name}}" + {{#validate_required?}} + if self[:{{key}}].nil? + raise ArgumentError, "Missing required EndpointParameter: :{{key}}" end - {{/required}} + {{/validate_required?}} {{/parameters}} end + + def self.create(config, options={}) + new({ + {{#config_parameters}} + {{key}}: {{{value}}}, + {{/config_parameters}} + }.merge(options)) + end end end diff --git a/build_tools/aws-sdk-code-generator/templates/endpoints_module.mustache b/build_tools/aws-sdk-code-generator/templates/endpoints_module.mustache index a93a5860b8a..339c286a8e8 100644 --- a/build_tools/aws-sdk-code-generator/templates/endpoints_module.mustache +++ b/build_tools/aws-sdk-code-generator/templates/endpoints_module.mustache @@ -11,19 +11,21 @@ module {{module_name}} {{#endpoint_classes}} class {{name}} def self.build(context) - {{module_name}}::EndpointParameters.new( + {{module_name}}::EndpointParameters.create( + context.config, {{#parameters}} - {{#static_string?}} - {{key}}: "{{{value}}}", - {{/static_string?}} - {{^static_string?}} {{key}}: {{{value}}}, - {{/static_string?}} {{/parameters}} ) end end {{/endpoint_classes}} + + OPERATION_PARAM_MAP = { + {{#endpoint_classes}} + {{operation}}: {{name}}, + {{/endpoint_classes}} + }.freeze end end diff --git a/build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache b/build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache index d9647844250..63279c3bd6b 100644 --- a/build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache +++ b/build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache @@ -86,12 +86,8 @@ The endpoint provider used to resolve endpoints. Any object that responds to end def parameters_for_operation(context) - case context.operation_name - {{#endpoint_classes}} - when :{{operation_name}} - {{module_name}}::Endpoints::{{class_name}}.build(context) - {{/endpoint_classes}} - end + operation_class = {{module_name}}::Endpoints::OPERATION_PARAM_MAP[context.operation_name] + operation_class ? operation_class.build(context) : {{module_name}}::EndpointParameters.create(context.config) end end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb index a0b64463658..c412715112a 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb @@ -103,9 +103,10 @@ def initialize_default_endpoint(client) if client.config.respond_to?(:endpoint_provider) && client_module.const_defined?(:EndpointParameters) && (param_class = client_module.const_get(:EndpointParameters)) && + param_class.respond_to?(:create) && (endpoint_provider = client.config.endpoint_provider) - params = build_parameters(param_class.new, client.config) + params = param_class.create(client.config) begin endpoint = endpoint_provider.resolve_endpoint(params) client.config.endpoint = endpoint.url @@ -119,24 +120,6 @@ def initialize_default_endpoint(client) end end - def build_parameters(params, config) - # generic parameters set for most (but not all) services - params.region = config.region if params.respond_to?(:region=) - params.use_dual_stack = config.use_dualstack_endpoint if params.respond_to?(:use_dual_stack=) - params.use_fips = config.use_fips_endpoint if params.respond_to?(:use_fips=) - - # service specific - if config.respond_to?(:sts_regional_endpoints) && params.respond_to?(:use_global_endpoint=) - params.use_global_endpoint = (config.sts_regional_endpoints == 'legacy') - end - - if config.respond_to?(:s3_us_east_1_regional_endpoint) && params.respond_to?(:use_global_endpoint=) - params.use_global_endpoint = (config.s3_us_east_1_regional_endpoint == 'legacy') - end - - params - end - # set a default endpoint in config using legacy (endpoints.json) resolver def resolve_legacy_endpoint(cfg) endpoint_prefix = cfg.api.metadata['endpointPrefix'] From 8f903a33543fe8c0b23c1a168eb433a6d72076d8 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Tue, 15 Oct 2024 19:03:25 -0700 Subject: [PATCH 17/18] PR cleanups --- .../endpoint_parameter.rb | 218 +++++++++--------- .../views/endpoint_parameters_class.rb | 3 +- .../views/endpoints_module.rb | 28 +-- .../spec/interfaces/plugins/endpoints_spec.rb | 2 +- .../templates/endpoints_module.mustache | 20 +- .../templates/endpoints_plugin.mustache | 7 +- .../aws-sdk-core/plugins/regional_endpoint.rb | 26 +-- 7 files changed, 145 insertions(+), 159 deletions(-) diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/endpoint_parameter.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/endpoint_parameter.rb index f84637660ab..36d22c08d52 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/endpoint_parameter.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/endpoint_parameter.rb @@ -2,10 +2,10 @@ module AwsSdkCodeGenerator class EndpointParameter - def initialize(name, definition, value, source) + def initialize(name, definition, service, operation=nil) @name = name - @source = source - @value = value + + @value, @source = endpoint_parameter_value(name, definition, service, operation) @type = definition['type'] @built_in = definition['builtIn'] @@ -59,142 +59,132 @@ def default? def validate_required? required && !default? end + def key Underscore.underscore(name) end - class << self - - def operation_specific_params?(service, operation) - parameters = service.endpoint_rules.fetch('parameters', {}) - parameters.each do |param_name, param_data| - _value, source = endpoint_parameter_value(service, param_name, param_data, operation) - return true if source == 'operation' - end - false - end - - # Most to least - # staticContextParams - # contextParam - # operationContextParams - # clientContextParams (always sourced from config) - # Built-In Bindings (sourced from config in most cases, context in some cases to allow operation level overrides) - # Built-in binding default values - # @retyrn [value, source]. source may be one of [operation, config, default] - def endpoint_parameter_value(service, param_name, param_data, operation=nil) - unless operation.nil? - value, source = [ - static_context_param(operation, param_name), 'operation' - ] - value, source = [ - context_param_value(service, operation, param_name), 'operation' - ] unless value - value, source = [ - operation_context_param_value(operation, param_name), 'operation' - ] unless value - end - + private + + # Most to least + # staticContextParams + # contextParam + # operationContextParams + # clientContextParams (always sourced from config) + # Built-In Bindings (sourced from config in most cases, context in some cases to allow operation level overrides) + # Built-in binding default values + # @retyrn [value, source]. source may be one of [operation, config, default] + def endpoint_parameter_value(param_name, param_data, service, operation) + unless operation.nil? + value, source = [ + static_context_param(operation, param_name), 'operation' + ] value, source = [ - client_context_param_value(service, param_name, param_data), - 'config' + context_param_value(service, operation, param_name), 'operation' ] unless value + value, source = [ + operation_context_param_value(operation, param_name), 'operation' + ] unless value + end + value, source = [ + client_context_param_value(service, param_name, param_data), + 'config' + ] unless value - # built-ins may be sourced from operation context in some cases - unless value - value, source = built_in_param_value(service, param_data) - end - [value || 'nil', source || 'default'] + # built-ins may be sourced from operation context in some cases + unless value + value, source = built_in_param_value(service, param_data) end - def client_context_param_value(service, param_name, param_data) - if service.api['clientContextParams']&.key?(param_name) && - !param_data['builtIn'] - "config.#{Underscore.underscore(param_name)}" - end + [value || 'nil', source || 'default'] + end + + def client_context_param_value(service, param_name, param_data) + if service.api['clientContextParams']&.key?(param_name) && + !param_data['builtIn'] + "config.#{Underscore.underscore(param_name)}" end + end - # built-ins may be sourced from operation context in some cases - def built_in_param_value(service, param_data) - source = 'config' - value = - case param_data['builtIn'] - when 'AWS::Region' - 'config.region' - when 'AWS::UseFIPS' - 'config.use_fips_endpoint' - when 'AWS::UseDualStack' - if service.name == 'S3' || service.name == 'S3Control' - source = 'operation' - 'context[:use_dualstack_endpoint]' - else - 'config.use_dualstack_endpoint' - end - when 'AWS::Auth::AccountId' - 'config.credentials.credentials.account_id' - when 'AWS::Auth::AccountIdEndpointMode' - 'config.account_id_endpoint_mode' - when 'AWS::STS::UseGlobalEndpoint' - "config.sts_regional_endpoints == 'legacy'" - when 'AWS::S3::UseGlobalEndpoint' - "config.s3_us_east_1_regional_endpoint == 'legacy'" - when 'AWS::S3::Accelerate' - if service.name == 'S3' || service.name == 'S3Control' - source = 'operation' - 'context[:use_accelerate_endpoint]' - else - 'config.use_accelerate_endpoint' - end - when 'AWS::S3::ForcePathStyle' - 'config.force_path_style' - when 'AWS::S3::UseArnRegion', 'AWS::S3Control::UseArnRegion' - 'config.s3_use_arn_region' - when 'AWS::S3::DisableMultiRegionAccessPoints' - 'config.s3_disable_multiregion_access_points' - when 'SDK::Endpoint' - 'config.regional_endpoint ? nil : config.endpoint.to_s' + # built-ins may be sourced from operation context in some cases + def built_in_param_value(service, param_data) + source = 'config' + value = + case param_data['builtIn'] + when 'AWS::Region' + 'config.region' + when 'AWS::UseFIPS' + 'config.use_fips_endpoint' + when 'AWS::UseDualStack' + if service.name == 'S3' || service.name == 'S3Control' + source = 'operation' + 'context[:use_dualstack_endpoint]' else - source = nil - nil # no value, not a default + 'config.use_dualstack_endpoint' end - [value, source] - end + when 'AWS::Auth::AccountId' + 'config.credentials.credentials.account_id' + when 'AWS::Auth::AccountIdEndpointMode' + 'config.account_id_endpoint_mode' + when 'AWS::STS::UseGlobalEndpoint' + "config.sts_regional_endpoints == 'legacy'" + when 'AWS::S3::UseGlobalEndpoint' + "config.s3_us_east_1_regional_endpoint == 'legacy'" + when 'AWS::S3::Accelerate' + if service.name == 'S3' || service.name == 'S3Control' + source = 'operation' + 'context[:use_accelerate_endpoint]' + else + 'config.use_accelerate_endpoint' + end + when 'AWS::S3::ForcePathStyle' + 'config.force_path_style' + when 'AWS::S3::UseArnRegion', 'AWS::S3Control::UseArnRegion' + 'config.s3_use_arn_region' + when 'AWS::S3::DisableMultiRegionAccessPoints' + 'config.s3_disable_multiregion_access_points' + when 'SDK::Endpoint' + '(config.endpoint.to_s unless config.regional_endpoint)' + else + source = nil + nil # no value, not a default + end + [value, source] + end - def context_param_value(service, operation, param_name) - return nil unless operation['input'] + def context_param_value(service, operation, param_name) + return nil unless operation['input'] - input_shape = operation['input']['shape'] - members = service.api['shapes'][input_shape].fetch('members', {}) - members.detect do |(member_name, member)| - context_param = member.fetch('contextParam', {}) - if context_param.fetch('name', nil) == param_name - break "context.params[:#{Underscore.underscore(member_name)}]" - end + input_shape = operation['input']['shape'] + members = service.api['shapes'][input_shape].fetch('members', {}) + members.detect do |(member_name, member)| + context_param = member.fetch('contextParam', {}) + if context_param.fetch('name', nil) == param_name + break "context.params[:#{Underscore.underscore(member_name)}]" end end + end - def operation_context_param_value(operation, param_name) - return nil unless operation['input'] + def operation_context_param_value(operation, param_name) + return nil unless operation['input'] - binding = operation.fetch('operationContextParams', {})[param_name] + binding = operation.fetch('operationContextParams', {})[param_name] - return nil unless binding + return nil unless binding - "JMESPath.search(\"#{Underscore.underscore_jmespath(binding['path'])}\", context.params)" - end + "JMESPath.search(\"#{Underscore.underscore_jmespath(binding['path'])}\", context.params)" + end - def static_context_param(operation, param_name) - value = operation.fetch('staticContextParams', {}) - .fetch(param_name, {}).fetch('value', nil) - if !value.nil? && value.is_a?(String) + def static_context_param(operation, param_name) + value = operation.fetch('staticContextParams', {}) + .fetch(param_name, {}).fetch('value', nil) + if !value.nil? && value.is_a?(String) "\"#{value}\"" - else - value - end + else + value end end end - -end \ No newline at end of file +end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoint_parameters_class.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoint_parameters_class.rb index 7c17a98dbff..70843b1519c 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoint_parameters_class.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoint_parameters_class.rb @@ -9,8 +9,7 @@ def initialize(options) @service = options.fetch(:service) if (parameters = @service.endpoint_rules&.fetch('parameters')) @parameters = parameters.map do |k,p| - value, source = EndpointParameter.endpoint_parameter_value(@service, k, p) - EndpointParameter.new(k, p, value, source) + EndpointParameter.new(k, p, @service) end end end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoints_module.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoints_module.rb index a409635e43c..16d9c4298c3 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoints_module.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/endpoints_module.rb @@ -9,13 +9,14 @@ def initialize(options) @parameters = @service.endpoint_rules.fetch('parameters', {}) - @endpoint_classes = @service.api['operations'].map do - |name, op| - EndpointClass.new( + @endpoint_classes = @service.api['operations'].each.with_object([]) do + |(name, op), classes| + endpoint_class = EndpointClass.new( name: name, parameters: endpoint_parameters_for_operation(op) ) - end.reject { |ec| ec.parameters.empty? } + classes << endpoint_class unless endpoint_class.parameters.empty? + end end # @return [Array] @@ -31,6 +32,10 @@ def module_name @service.module_name end + def operation_specific_parameters? + @endpoint_classes.empty? + end + class EndpointClass def initialize(options) @name = options[:name] @@ -51,18 +56,15 @@ def initialize(options) private def endpoint_parameters_for_operation(operation) - @parameters.map do |param_name, param_data| - value, source = EndpointParameter.endpoint_parameter_value( - @service, param_name, param_data, operation - ) - - EndpointParameter.new( + @parameters.each.with_object([]) do |(param_name, param_data), parameters| + p = EndpointParameter.new( param_name, param_data, - value, - source + @service, + operation ) - end.select { |p| p.source == 'operation' } + parameters << p if p.source == 'operation' + end end end end diff --git a/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb b/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb index 4dab07d5282..20fe230986a 100644 --- a/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb +++ b/build_tools/aws-sdk-code-generator/spec/interfaces/plugins/endpoints_spec.rb @@ -126,7 +126,7 @@ end before(:each) do - client # force client initialization before testing resolution + client # force client initialization before testing resolution to account for config.endpoint end # Most to least diff --git a/build_tools/aws-sdk-code-generator/templates/endpoints_module.mustache b/build_tools/aws-sdk-code-generator/templates/endpoints_module.mustache index 339c286a8e8..91e460352fe 100644 --- a/build_tools/aws-sdk-code-generator/templates/endpoints_module.mustache +++ b/build_tools/aws-sdk-code-generator/templates/endpoints_module.mustache @@ -22,10 +22,20 @@ module {{module_name}} {{/endpoint_classes}} - OPERATION_PARAM_MAP = { - {{#endpoint_classes}} - {{operation}}: {{name}}, - {{/endpoint_classes}} - }.freeze + def self.parameters_for_operation(context) + {{#operation_specific_parameters?}} + {{module_name}}::EndpointParameters.create(context.config) + {{/operation_specific_parameters?}} + {{^operation_specific_parameters?}} + case context.operation_name + {{#endpoint_classes}} + when :{{operation}} + {{name}}.build(context) + {{/endpoint_classes}} + else + {{module_name}}::EndpointParameters.create(context.config) + end + {{/operation_specific_parameters?}} + end end end diff --git a/build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache b/build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache index 63279c3bd6b..42b84453054 100644 --- a/build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache +++ b/build_tools/aws-sdk-code-generator/templates/endpoints_plugin.mustache @@ -34,7 +34,7 @@ The endpoint provider used to resolve endpoints. Any object that responds to class Handler < Seahorse::Client::Handler def call(context) unless context[:discovered_endpoint] - params = parameters_for_operation(context) + params = {{module_name}}::Endpoints.parameters_for_operation(context) endpoint = context.config.endpoint_provider.resolve_endpoint(params) context.http_request.endpoint = endpoint.url @@ -84,11 +84,6 @@ The endpoint provider used to resolve endpoints. Any object that responds to context.http_request.headers[key] = value end end - - def parameters_for_operation(context) - operation_class = {{module_name}}::Endpoints::OPERATION_PARAM_MAP[context.operation_name] - operation_class ? operation_class.build(context) : {{module_name}}::EndpointParameters.create(context.config) - end end def add_handlers(handlers, _config) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb index c412715112a..5fc7552c15d 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb @@ -100,24 +100,14 @@ def after_initialize(client) def initialize_default_endpoint(client) client_module = Object.const_get(client.class.name.rpartition('::').first) - if client.config.respond_to?(:endpoint_provider) && - client_module.const_defined?(:EndpointParameters) && - (param_class = client_module.const_get(:EndpointParameters)) && - param_class.respond_to?(:create) && - (endpoint_provider = client.config.endpoint_provider) - - params = param_class.create(client.config) - begin - endpoint = endpoint_provider.resolve_endpoint(params) - client.config.endpoint = endpoint.url - rescue ArgumentError - # fallback to legacy - client.config.endpoint = resolve_legacy_endpoint(client.config) - end - else - # fallback to legacy - client.config.endpoint = resolve_legacy_endpoint(client.config) - end + param_class = client_module.const_get(:EndpointParameters) + endpoint_provider = client.config.endpoint_provider + params = param_class.create(client.config) + endpoint = endpoint_provider.resolve_endpoint(params) + client.config.endpoint = endpoint.url + rescue ArgumentError, NameError + # fallback to legacy + client.config.endpoint = resolve_legacy_endpoint(client.config) end # set a default endpoint in config using legacy (endpoints.json) resolver From 01a729c66891f72abf1d3a617be86d545d655743 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 16 Oct 2024 10:49:39 -0700 Subject: [PATCH 18/18] Bump mincore versions --- build_tools/services.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/services.rb b/build_tools/services.rb index 045bb254e2a..67a5ed670c5 100644 --- a/build_tools/services.rb +++ b/build_tools/services.rb @@ -10,10 +10,10 @@ class ServiceEnumerator MANIFEST_PATH = File.expand_path('../../services.json', __FILE__) # Minimum `aws-sdk-core` version for new gem builds - MINIMUM_CORE_VERSION = "3.207.0" + MINIMUM_CORE_VERSION = "3.210.0" # Minimum `aws-sdk-core` version for new S3 gem builds - MINIMUM_CORE_VERSION_S3 = "3.207.0" + MINIMUM_CORE_VERSION_S3 = "3.210.0" EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration"