Reduce memory usage from aws-partitions

gems/aws-partitions/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - Add partition metadata module, allowing access without loading entire partitions.json.
+
 1.987.0 (2024-10-08)
 ------------------
 

gems/aws-partitions/lib/aws-partitions.rb

@@ -5,6 +5,7 @@
 require_relative 'aws-partitions/partition_list'
 require_relative 'aws-partitions/region'
 require_relative 'aws-partitions/service'
+require_relative 'aws-partitions/metadata'
 
 require 'json'
 

gems/aws-partitions/lib/aws-partitions/metadata.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Aws
+  module Partitions
+    # @api private
+    module Metadata
+      class << self
+
+        # aws.partition(region: string) Option<Partition>
+        def partition(region)
+          partition =
+            partitions.find { |p| p['regions']&.fetch(region, nil) } ||
+            partitions.find { |p| region.match(p['regionRegex']) } ||
+            partitions.find { |p| p['id'] == 'aws' }
+
+          return nil unless partition
+
+          partition['outputs']
+        end
+
+        def partitions
+          @partitions ||= default_partition_metadata
+        end
+
+        def default_partition_metadata
+          path = File.expand_path('../../../partitions-metadata.json', __FILE__)
+          JSON.parse(File.read(path), freeze: true)['partitions']
+        end
+      end
+    end
+  end
+end

gems/aws-sdk-core/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - reduce memory usage by not using legacy endpoint data unless required.
+
 3.209.1 (2024-09-25)
 ------------------
 

gems/aws-sdk-core/aws-sdk-core.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.files         = Dir['LICENSE.txt', 'CHANGELOG.md', 'VERSION', 'lib/**/*.rb', 'sig/**/*.rbs', 'ca-bundle.crt']
 
   spec.add_dependency('jmespath', '~> 1', '>= 1.6.1') # necessary for secure jmespath JSON parsing
-  spec.add_dependency('aws-partitions', '~> 1', '>= 1.651.0') # necessary for new endpoint resolution
+  spec.add_dependency('aws-partitions', '~> 1', '>= 1.988.0') # necessary for new endpoint resolution
   spec.add_dependency('aws-sigv4', '~> 1.9') # necessary for s3 express auth/native sigv4a support
   spec.add_dependency('aws-eventstream', '~> 1', '>= 1.3.0') # necessary for binary eventstream
 

gems/aws-sdk-core/lib/aws-sdk-core/endpoints/matchers.rb

@@ -94,14 +94,7 @@ def self.valid_host_label?(value, allow_sub_domains = false)
 
       # aws.partition(value: string) Option<Partition>
       def self.aws_partition(value)
-        partition =
-          Aws::Partitions.find { |p| p.region?(value) } ||
-          Aws::Partitions.find { |p| value.match(p.region_regex) } ||
-          Aws::Partitions.find { |p| p.name == 'aws' }
-
-        return nil unless partition
-
-        partition.metadata
+        Aws::Partitions::Metadata.partition(value)
       end
 
       # aws.parseArn(value: string) Option<ARN>

gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -20,7 +20,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
 * `ENV['AWS_DEFAULT_REGION']`
 * `~/.aws/credentials`
 * `~/.aws/config`
-        DOCS
+             DOCS
         resolve_region(cfg)
       end
 
@@ -35,7 +35,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
 * `Aws.config[:sigv4a_signing_region_set]`
 * `ENV['AWS_SIGV4A_SIGNING_REGION_SET']`
 * `~/.aws/config`
-        DOCS
+             DOCS
         resolve_sigv4a_signing_region_set(cfg)
       end
 
@@ -44,7 +44,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
         docstring: <<-DOCS) do |cfg|
 When set to `true`, dualstack enabled endpoints (with `.aws` TLD)
 will be used if available.
-        DOCS
+             DOCS
         resolve_use_dualstack_endpoint(cfg)
       end
 
@@ -54,7 +54,7 @@ class RegionalEndpoint < Seahorse::Client::Plugin
 When set to `true`, fips compatible endpoints will be used if available.
 When a `fips` region is used, the region is normalized and this config
 is set to `true`.
-        DOCS
+             DOCS
         resolve_use_fips_endpoint(cfg)
       end
 
@@ -67,15 +67,15 @@ class RegionalEndpoint < Seahorse::Client::Plugin
         docstring: <<-DOCS) do |cfg|
 Setting to true disables use of endpoint URLs provided via environment 
 variables and the shared configuration file.
-        DOCS
+             DOCS
         resolve_ignore_configured_endpoint_urls(cfg)
       end
 
       option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg|
 The client endpoint is normally constructed from the `:region`
 option. You should only configure an `:endpoint` when connecting
 to test or custom endpoints. This should be a valid HTTP(S) URI.
-        DOCS
+      DOCS
         resolve_endpoint(cfg)
       end
 
@@ -150,7 +150,18 @@ def resolve_endpoint(cfg)
           # that a custom endpoint has NOT been configured by the user
           cfg.override_config(:regional_endpoint, true)
 
-          resolve_legacy_endpoint(cfg)
+          # preserve legacy (pre EP2) client.config.endpoint still resolves a value
+          # when needed.
+          struct = cfg.instance_variable_get(:@struct)
+          if struct
+            # need a separate proc to get namespace/private access to resolve_legacy_endpoint
+            b = proc { resolve_legacy_endpoint(struct) }
+            struct.define_singleton_method(:endpoint) { @endpoint ||= b.call }
+            nil
+          else
+            # backup in case internal details of config resolver change
+            resolve_legacy_endpoint(cfg)
+          end
         end
 
         # get a custom configured endpoint from ENV or configuration
@@ -213,7 +224,7 @@ def resolve_legacy_endpoint(cfg)
             sts_regional = cfg.sts_regional_endpoints
           end
 
-          Aws::Partitions::EndpointProvider.resolve(
+          endpoint = Aws::Partitions::EndpointProvider.resolve(
             cfg.region,
             endpoint_prefix,
             sts_regional,
@@ -222,6 +233,7 @@ def resolve_legacy_endpoint(cfg)
               fips: cfg.use_fips_endpoint
             }
           )
+          URI(endpoint)
         end
       end
     end

gems/aws-sdk-core/lib/seahorse/client/plugins/endpoint.rb

@@ -24,12 +24,10 @@ def add_handlers(handlers, config)
         end
 
         def after_initialize(client)
-          endpoint = client.config.endpoint
-          if endpoint.nil?
-            msg = "missing required option `:endpoint'"
-            raise ArgumentError, msg
-          end
+          # validate endpoint only if user set a custom endpoint
+          return if client.config.respond_to?(:regional_endpoint) && client.config.regional_endpoint
 
+          endpoint = client.config.endpoint
           endpoint = URI.parse(endpoint.to_s)
           if URI::HTTPS === endpoint or URI::HTTP === endpoint
             client.config.endpoint = endpoint
@@ -42,7 +40,11 @@ def after_initialize(client)
         class Handler < Client::Handler
 
           def call(context)
-            context.http_request.endpoint = URI.parse(context.config.endpoint.to_s)
+            if !context.config.respond_to?(:regional_endpoint) || # generic
+               !context.config.regional_endpoint || # custom endpoint
+               !context.config.respond_to?(:endpoint_provider) # legacy/pre-ep2/no rules
+              context.http_request.endpoint = URI.parse(context.config.endpoint.to_s)
+            end
             @handler.call(context)
           end