From 81dbde7b22d4eed448e978afbe487087529bd856 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Wed, 10 Jan 2024 14:04:39 -0500 Subject: [PATCH 01/19] Support S3 Access Grants --- build_tools/services.rb | 2 +- gems/aws-sdk-core/CHANGELOG.md | 2 + gems/aws-sdk-core/lib/aws-sdk-core.rb | 1 + .../lib/aws-sdk-core/lru_cache.rb | 72 +++++++++ gems/aws-sdk-s3/CHANGELOG.md | 4 + gems/aws-sdk-s3/aws-sdk-s3.gemspec | 1 + gems/aws-sdk-s3/features/env.rb | 1 + gems/aws-sdk-s3/lib/aws-sdk-s3.rb | 1 + .../aws-sdk-s3/access_grants_credentials.rb | 52 ++++++ .../access_grants_credentials_provider.rb | 149 ++++++++++++++++++ gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb | 8 + .../lib/aws-sdk-s3/customizations.rb | 4 + .../lib/aws-sdk-s3/plugins/access_grants.rb | 76 +++++++++ gems/aws-sdk-s3/spec/spec_helper.rb | 1 + services.json | 2 + 15 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 gems/aws-sdk-core/lib/aws-sdk-core/lru_cache.rb create mode 100644 gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb create mode 100644 gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb create mode 100644 gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb diff --git a/build_tools/services.rb b/build_tools/services.rb index c722dc5c8de..64e11062128 100644 --- a/build_tools/services.rb +++ b/build_tools/services.rb @@ -13,7 +13,7 @@ class ServiceEnumerator MINIMUM_CORE_VERSION = "3.188.0" # Minimum `aws-sdk-core` version for new S3 gem builds - MINIMUM_CORE_VERSION_S3 = "3.189.0" + MINIMUM_CORE_VERSION_S3 = "3.191.0" EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration" diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index c46a4bb0788..f4a7f7b1b34 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Add an API private cache for S3 Express and Access Grants. + 3.190.1 (2023-12-20) ------------------ diff --git a/gems/aws-sdk-core/lib/aws-sdk-core.rb b/gems/aws-sdk-core/lib/aws-sdk-core.rb index 28cecc49671..09f5780a91b 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core.rb @@ -96,6 +96,7 @@ require_relative 'aws-sdk-core/arn' require_relative 'aws-sdk-core/arn_parser' require_relative 'aws-sdk-core/ec2_metadata' +require_relative 'aws-sdk-core/lru_cache' # dynamic endpoints require_relative 'aws-sdk-core/endpoints' diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/lru_cache.rb b/gems/aws-sdk-core/lib/aws-sdk-core/lru_cache.rb new file mode 100644 index 00000000000..0f4872d9000 --- /dev/null +++ b/gems/aws-sdk-core/lib/aws-sdk-core/lru_cache.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Aws + # @api private + # A simple thread safe LRU cache + class LRUCache + def initialize(options = {}) + @max_entries = options[:max_entries] || 100 + @expiration = options[:expiration] + @entries = {} + @mutex = Mutex.new + end + + # @param [String] key + # @return [Object] + def [](key) + @mutex.synchronize do + value = @entries[key] + if value + @entries.delete(key) + @entries[key] = value unless value.expired? + end + @entries[key]&.value + end + end + + # @param [String] key + # @param [Object] value + def []=(key, value) + @mutex.synchronize do + @entries.shift unless @entries.size < @max_entries + # delete old value if exists + @entries.delete(key) + @entries[key] = Entry.new(value: value, expiration: @expiration) + @entries[key].value + end + end + + # @param [String] key + # @return [Boolean] + def key?(key) + @mutex.synchronize do + @entries.delete(key) if @entries.key?(key) && @entries[key].expired? + @entries.key?(key) + end + end + + def clear + @mutex.synchronize do + @entries.clear + end + end + + # @api private + class Entry + def initialize(options = {}) + @value = options[:value] + @expiration = options[:expiration] + @created_time = Time.now + end + + # @return [Object] + attr_reader :value + + def expired? + return false unless @expiration + + Time.now - @created_time > @expiration + end + end + end +end diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index c01e327f1ef..eaa27a57185 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,6 +1,10 @@ Unreleased Changes ------------------ +* Feature - Support S3 Access Grants authentication. Access Grants can be enabled with the `s3_access_grants` option, and custom options can be passed into the `access_grants_credentials_provider` option. + +* Feature - S3 now depends on `aws-sdk-s3control` for Access Grants credential fetching. + 1.142.0 (2023-12-22) ------------------ diff --git a/gems/aws-sdk-s3/aws-sdk-s3.gemspec b/gems/aws-sdk-s3/aws-sdk-s3.gemspec index d8b81cb5be5..e04c0b32ee6 100644 --- a/gems/aws-sdk-s3/aws-sdk-s3.gemspec +++ b/gems/aws-sdk-s3/aws-sdk-s3.gemspec @@ -26,6 +26,7 @@ Gem::Specification.new do |spec| } spec.add_dependency('aws-sdk-kms', '~> 1') + spec.add_dependency('aws-sdk-s3control', '~> 1.73') spec.add_dependency('aws-sigv4', '~> 1.8') spec.add_dependency('aws-sdk-core', '~> 3', '>= 3.189.0') diff --git a/gems/aws-sdk-s3/features/env.rb b/gems/aws-sdk-s3/features/env.rb index 2d204f85107..138b4f3cd53 100644 --- a/gems/aws-sdk-s3/features/env.rb +++ b/gems/aws-sdk-s3/features/env.rb @@ -10,6 +10,7 @@ $:.unshift(File.expand_path('../../lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-core/features', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-kms/lib', __FILE__)) +$:.unshift(File.expand_path('../../../aws-sdk-s3control/lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sigv4/lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-core/lib', __FILE__)) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3.rb index 86f83bfb787..758162ac142 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3.rb @@ -9,6 +9,7 @@ require 'aws-sdk-kms' +require 'aws-sdk-s3control' require 'aws-sigv4' require 'aws-sdk-core' diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb new file mode 100644 index 00000000000..47115817a7d --- /dev/null +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'set' + +module Aws + module S3 + # @api private + class AccessGrantsCredentials + include CredentialProvider + include RefreshingCredentials + + def initialize(options = {}) + @client = options[:client] + @get_data_access_params = {} + options.each_pair do |key, value| + if self.class.get_data_access_options.include?(key) + @get_data_access_params[key] = value + end + end + @async_refresh = true + super + end + + # @return [S3Control::Client] + attr_reader :client + + private + + def refresh + c = @client.get_data_access(@get_data_access_params).credentials + @credentials = Credentials.new( + c.access_key_id, + c.secret_access_key, + c.session_token + ) + @expiration = c.expiration + end + + class << self + + # @api private + def get_data_access_options + @gdao ||= begin + input = S3Control::Client.api.operation(:get_data_access).input + Set.new(input.shape.member_names) + end + end + + end + end + end +end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb new file mode 100644 index 00000000000..8ba2697cde8 --- /dev/null +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module Aws + module S3 + # @api private + ACCESS_GRANTS_CREDENTIALS_CACHE = LRUCache.new + # @api private + ACCESS_GRANTS_ACCOUNT_ID_CACHE = LRUCache.new(expiration: 60 * 10) + + # Returns Credentials class for S3 Access Grants. Accepts GetDataAccess + # params and other configuration as options. See + # {Aws::S3Control::Client#get_data_access} for details. + class AccessGrantsCredentialsProvider + # @param [Hash] options + # @option options [Client] :client The S3 Control client used to create + # the session. + # @option options [String] :privilege ('Default') The privilege to use + # when requesting credentials. (see: {Aws::S3Control::Client#get_data_access}) + # @option options [Boolean] :fallback (false) When true, if access is + # denied, the provider will fall back to the configured credentials. + # @option options [Boolean] :caching (true) When true, credentials and + # bucket account ids will be cached. + # @option options [Callable] :before_refresh Proc called before + # credentials are refreshed. + def initialize(options = {}) + @client = options.delete(:client) || S3Control::Client.new + @fallback = options.delete(:fallback) || false + @caching = options.delete(:caching) || true + @options = options + return unless @caching + + @credentials_cache = ACCESS_GRANTS_CREDENTIALS_CACHE + @account_id_cache = ACCESS_GRANTS_ACCOUNT_ID_CACHE + end + + def access_grants_credentials_for(options = {}) + target = target_prefix( + options[:bucket], + options[:key], + options[:prefix] + ) + + if @caching + cached_credentials_for( + target, + options[:credentials], + options[:permission] + ) + else + new_credentials_for( + target, + options[:credentials], + options[:permission] + ) + end + rescue Aws::S3Control::Errors::AccessDenied + raise unless @fallback + + warn 'Access denied for S3 Access Grants. Falling back to ' \ + 'configured credentials.' + options[:credentials] + end + + private + + def cached_credentials_for(target, credentials, permission) + key = credentials_cache_key(target, credentials, permission) + + if @credentials_cache.key?(key) + @credentials_cache[key] + else + @credentials_cache[key] = new_credentials_for( + target, + credentials, + permission + ) + end + end + + def new_credentials_for(target, credentials, permission) + AccessGrantsCredentials.new( + target: target, + account_id: account_id_for_access_grants(target, credentials), + permission: permission, + client: @client, + **@options + ) + end + + def account_id_for_access_grants(target, credentials) + if @caching + cached_account_id_for(target, credentials) + else + new_account_id_for(target, credentials) + end + end + + def cached_account_id_for(target, credentials) + key = account_id_cache_key(target) + + if @account_id_cache.key?(key) + @account_id_cache[key] + else + @account_id_cache[key] = new_account_id_for(target, credentials) + end + end + + # returns the account id associated with the access grants instance + def new_account_id_for(target, credentials) + resp = @client.get_access_grants_instance_for_prefix( + s3_prefix: target, + account_id: account_id_for_credentials(credentials), + ) + ARNParser.parse(resp.access_grants_instance_arn).account_id + end + + # returns the account id for the configured credentials + def account_id_for_credentials(credentials) + if credentials.respond_to?(:account_id) && credentials.account_id && + !credentials.account_id.empty? + credentials.account_id + else + Aws::STS::Client.new.get_caller_identity.account + end + end + + def target_prefix(bucket, key, prefix) + if key && !key.empty? + "s3://#{bucket}/#{key}" + elsif prefix && !prefix.empty? + "s3://#{bucket}/#{prefix}" + else + "s3://#{bucket}/*" + end + end + + # TODO - should check multiple targets and permissions in a smart way + def credentials_cache_key(target, credentials, permission) + "#{credentials.access_key_id}-#{credentials.secret_access_key}" \ + "-#{permission}-#{target}" + end + + # uses bucket name from target prefix + def account_id_cache_key(target) + URI(target).host + end + end + end +end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb index 4e826c2ddf5..10d95493959 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb @@ -34,6 +34,7 @@ require 'aws-sdk-core/plugins/sign.rb' require 'aws-sdk-core/plugins/protocols/rest_xml.rb' require 'aws-sdk-s3/plugins/accelerate.rb' +require 'aws-sdk-s3/plugins/access_grants.rb' require 'aws-sdk-s3/plugins/arn.rb' require 'aws-sdk-s3/plugins/bucket_dns.rb' require 'aws-sdk-s3/plugins/bucket_name_restrictions.rb' @@ -104,6 +105,7 @@ class Client < Seahorse::Client::Base add_plugin(Aws::Plugins::Sign) add_plugin(Aws::Plugins::Protocols::RestXml) add_plugin(Aws::S3::Plugins::Accelerate) + add_plugin(Aws::S3::Plugins::AccessGrants) add_plugin(Aws::S3::Plugins::ARN) add_plugin(Aws::S3::Plugins::BucketDns) add_plugin(Aws::S3::Plugins::BucketNameRestrictions) @@ -366,6 +368,12 @@ class Client < Seahorse::Client::Base # in the future. # # + # @option options [Boolean] :s3_access_grants (false) + # TODO + # + # @option options [Aws::S3::AccessGrantsCredentialsProvider] :s3_access_grants_credentials_provider + # TODO + # # @option options [Boolean] :s3_disable_multiregion_access_points (false) # When set to `false` this will option will raise errors when multi-region # access point ARNs are used. Multi-region access points can potentially diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb index 4bc5b2fbb36..20289f484d2 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb @@ -21,6 +21,10 @@ require 'aws-sdk-s3/express_credentials_cache' require 'aws-sdk-s3/express_credentials_provider' +# s3 access grants auth +require 'aws-sdk-s3/access_grants_credentials' +require 'aws-sdk-s3/access_grants_credentials_provider' + # customizations to generated classes require 'aws-sdk-s3/customizations/bucket' require 'aws-sdk-s3/customizations/errors' diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb new file mode 100644 index 00000000000..23cd682bd44 --- /dev/null +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Aws + module S3 + module Plugins + # @api private + class AccessGrants < Seahorse::Client::Plugin + option( + :s3_access_grants, + default: false, + doc_type: 'Boolean', + docstring: <<-DOCS) +When `true`, the S3 client will use the S3 Access Grants feature to +authenticate requests. Bucket credentials will be fetched from S3 +Control using the `get_data_access` API. + DOCS + + option(:access_grants_credentials_provider, + doc_type: 'Aws::S3::AccessGrantsCredentialsProvider', + docstring: <<-DOCS) do |_cfg| +When `s3_access_grants` is `true`, this option can be used to provide +additional options to the credentials provider, including a privilege +setting, caching, and fallback behavior. + DOCS + Aws::S3::AccessGrantsCredentialsProvider.new + end + + # @api private + class Handler < Seahorse::Client::Handler + PERMISSION_MAP = { + head_object: 'READ', + get_object: 'READ', + get_object_acl: 'READ', + list_multipart_uploads: 'READ', + list_objects_v2: 'READ', + list_object_versions: 'READ', + list_parts: 'READ', + put_object: 'WRITE', + put_object_acl: 'WRITE', + delete_object: 'WRITE', + abort_multipart_upload: 'WRITE', + create_multipart_upload: 'WRITE', + upload_part: 'WRITE', + complete_multipart_upload: 'WRITE' + }.freeze + + def call(context) + # only use access grants if it is a bucket operation. + # express session auth is not supported + if (params = context[:endpoint_params]) && params[:bucket] && + (permission = PERMISSION_MAP[context.operation_name]) && + context.config.disable_s3_express_session_auth + credentials_provider = + context.config.access_grants_credentials_provider + + credentials = credentials_provider.access_grants_credentials_for( + bucket: params[:bucket], + key: params[:key], + prefix: params[:prefix], + credentials: context.config.credentials.credentials, + permission: permission + ) + context[:sigv4_credentials] = credentials # Sign will use this + end + + @handler.call(context) + end + end + + def add_handlers(handlers,config) + handlers.add(Handler) if config.s3_access_grants + end + end + end + end +end diff --git a/gems/aws-sdk-s3/spec/spec_helper.rb b/gems/aws-sdk-s3/spec/spec_helper.rb index ac98b73f01e..ea53682b8da 100644 --- a/gems/aws-sdk-s3/spec/spec_helper.rb +++ b/gems/aws-sdk-s3/spec/spec_helper.rb @@ -11,6 +11,7 @@ $:.unshift(File.expand_path('../../lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-kms/lib', __FILE__)) +$:.unshift(File.expand_path('../../../aws-sdk-s3control/lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sigv4/lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-core/lib', __FILE__)) diff --git a/services.json b/services.json index 4b16cb52aea..42b7c1f3500 100644 --- a/services.json +++ b/services.json @@ -993,10 +993,12 @@ "models": "s3/2006-03-01", "dependencies": { "aws-sdk-kms": "~> 1", + "aws-sdk-s3control": "~> 1.73", "aws-sigv4": "~> 1.8" }, "addPlugins": [ "Aws::S3::Plugins::Accelerate", + "Aws::S3::Plugins::AccessGrants", "Aws::S3::Plugins::ARN", "Aws::S3::Plugins::BucketDns", "Aws::S3::Plugins::BucketNameRestrictions", From 68a0664f85d3a173a998b36d98240ed40bcc4094 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Wed, 10 Jan 2024 14:16:47 -0500 Subject: [PATCH 02/19] Use LRU cache for express credentials --- .../lib/aws-sdk-s3/customizations.rb | 1 - .../aws-sdk-s3/express_credentials_cache.rb | 30 ------------------- .../express_credentials_provider.rb | 9 +++++- 3 files changed, 8 insertions(+), 32 deletions(-) delete mode 100644 gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_cache.rb diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb index 20289f484d2..b16dca35841 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb @@ -18,7 +18,6 @@ # s3 express session auth require 'aws-sdk-s3/express_credentials' -require 'aws-sdk-s3/express_credentials_cache' require 'aws-sdk-s3/express_credentials_provider' # s3 access grants auth diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_cache.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_cache.rb deleted file mode 100644 index 96904818be9..00000000000 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_cache.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Aws - module S3 - # @api private - class ExpressCredentialsCache - def initialize - @credentials = {} - @mutex = Mutex.new - end - - def [](bucket_name) - @mutex.synchronize { @credentials[bucket_name] } - end - - def []=(bucket_name, credential_provider) - @mutex.synchronize do - @credentials[bucket_name] = credential_provider - end - end - - def clear - @mutex.synchronize { @credentials = {} } - end - end - - # @api private - EXPRESS_CREDENTIALS_CACHE = ExpressCredentialsCache.new - end -end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb index f909c4aedde..d822d058ae2 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb @@ -2,16 +2,23 @@ module Aws module S3 + # @api private + EXPRESS_CREDENTIALS_CACHE = LRUCache.new + # Returns Credentials class for S3 Express. Accepts CreateSession # params as options. See {Client#create_session} for details. class ExpressCredentialsProvider # @param [Hash] options - # @option options [Client] :client The S3 client used to create the session. + # @option options [Client] :client The S3 client used to create the + # session. # @option options [String] :session_mode (see: {Client#create_session}) + # @option options [Boolean] :caching (true) When true, credentials will + # be cached. # @option options [Callable] :before_refresh Proc called before # credentials are refreshed. def initialize(options = {}) @client = options.delete(:client) + @caching = options.delete(:caching) || true @options = options @cache = EXPRESS_CREDENTIALS_CACHE end From 7da33d515c7b1e9455b40435dc7d391963b72734 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Wed, 10 Jan 2024 15:22:21 -0500 Subject: [PATCH 03/19] Fix existing tests --- .../access_grants_credentials_provider.rb | 27 ++++++++++++---- .../lib/aws-sdk-s3/plugins/access_grants.rb | 32 ++++++++++++++----- .../plugins/express_session_auth.rb | 2 +- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb index 8ba2697cde8..1599e7014ee 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb @@ -12,8 +12,10 @@ module S3 # {Aws::S3Control::Client#get_data_access} for details. class AccessGrantsCredentialsProvider # @param [Hash] options - # @option options [Client] :client The S3 Control client used to create - # the session. + # @option options [Client] :s3_control_client The S3 Control client used + # to create the session. + # @option options [Client] :sts_client The STS client used for fetching + # the account ID for the credentials if not provided. # @option options [String] :privilege ('Default') The privilege to use # when requesting credentials. (see: {Aws::S3Control::Client#get_data_access}) # @option options [Boolean] :fallback (false) When true, if access is @@ -23,7 +25,8 @@ class AccessGrantsCredentialsProvider # @option options [Callable] :before_refresh Proc called before # credentials are refreshed. def initialize(options = {}) - @client = options.delete(:client) || S3Control::Client.new + @s3_control_client = options.delete(:s3_control_client) + @sts_client = options.delete(:sts_client) @fallback = options.delete(:fallback) || false @caching = options.delete(:caching) || true @options = options @@ -61,6 +64,9 @@ def access_grants_credentials_for(options = {}) options[:credentials] end + attr_accessor :s3_credentials + attr_accessor :s3_region + private def cached_credentials_for(target, credentials, permission) @@ -78,11 +84,16 @@ def cached_credentials_for(target, credentials, permission) end def new_credentials_for(target, credentials, permission) + @s3_control_client ||= Aws::S3Control::Client.new( + credentials: s3_credentials, + region: s3_region + ) + AccessGrantsCredentials.new( target: target, account_id: account_id_for_access_grants(target, credentials), permission: permission, - client: @client, + client: @s3_control_client, **@options ) end @@ -107,7 +118,7 @@ def cached_account_id_for(target, credentials) # returns the account id associated with the access grants instance def new_account_id_for(target, credentials) - resp = @client.get_access_grants_instance_for_prefix( + resp = @s3_control_client.get_access_grants_instance_for_prefix( s3_prefix: target, account_id: account_id_for_credentials(credentials), ) @@ -120,7 +131,11 @@ def account_id_for_credentials(credentials) !credentials.account_id.empty? credentials.account_id else - Aws::STS::Client.new.get_caller_identity.account + @sts_client ||= Aws::STS::Client.new( + credentials: s3_credentials, + region: s3_region + ) + @sts_client.get_caller_identity.account end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb index 23cd682bd44..62d4831122e 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb @@ -45,15 +45,13 @@ class Handler < Seahorse::Client::Handler }.freeze def call(context) - # only use access grants if it is a bucket operation. - # express session auth is not supported - if (params = context[:endpoint_params]) && params[:bucket] && - (permission = PERMISSION_MAP[context.operation_name]) && - context.config.disable_s3_express_session_auth - credentials_provider = - context.config.access_grants_credentials_provider + if access_grants_operation?(context) && + !s3_express_endpoint?(context) + params = context[:endpoint_params] + permission = PERMISSION_MAP[context.operation_name] - credentials = credentials_provider.access_grants_credentials_for( + provider = context.config.access_grants_credentials_provider + credentials = provider.access_grants_credentials_for( bucket: params[:bucket], key: params[:key], prefix: params[:prefix], @@ -65,11 +63,29 @@ def call(context) @handler.call(context) end + + private + + def access_grants_operation?(context) + params = context[:endpoint_params] + params[:bucket] && PERMISSION_MAP[context.operation_name] + end + + def s3_express_endpoint?(context) + props = context[:endpoint_properties] + props['backend'] == 'S3Express' + end end def add_handlers(handlers,config) handlers.add(Handler) if config.s3_access_grants end + + def after_initialize(client) + provider = client.config.access_grants_credentials_provider + provider.s3_credentials = client.config.credentials + provider.s3_region = client.config.region + end end end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/express_session_auth.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/express_session_auth.rb index 7b33dcde066..35bf53a6470 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/express_session_auth.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/express_session_auth.rb @@ -30,7 +30,7 @@ class Handler < Seahorse::Client::Handler def call(context) if (props = context[:endpoint_properties]) # S3 Express endpoint - turn off md5 and enable crc32 default - if (backend = props['backend']) && backend == 'S3Express' + if props['backend'] == 'S3Express' if context.operation_name == :put_object || checksum_required?(context) context[:default_request_checksum_algorithm] = 'CRC32' end From 8c20dc5ad0ac706f65b82a6f7aa54d1680a7eee0 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Tue, 6 Feb 2024 15:39:52 -0500 Subject: [PATCH 04/19] S3 Control regional client support --- .../access_grants_credentials_provider.rb | 114 ++++++++++++------ .../lib/aws-sdk-s3/bucket_region_cache.rb | 8 ++ .../lib/aws-sdk-s3/customizations/errors.rb | 17 ++- .../lib/aws-sdk-s3/plugins/access_grants.rb | 4 +- 4 files changed, 98 insertions(+), 45 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb index 1599e7014ee..acb3e4b5215 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb @@ -12,10 +12,17 @@ module S3 # {Aws::S3Control::Client#get_data_access} for details. class AccessGrantsCredentialsProvider # @param [Hash] options - # @option options [Client] :s3_control_client The S3 Control client used - # to create the session. - # @option options [Client] :sts_client The STS client used for fetching - # the account ID for the credentials if not provided. + # @option options [Hash] :s3_control_client_options The S3 Control + # client options used to create regional S3 Control clients to + # create the session. Region will be set to the region of the + # bucket. + # @option options [Aws::STS::Client] :sts_client The STS client used for + # fetching the Account ID for the credentials if credentials do not + # include an Account ID. + # @option options [Aws::S3::Client] :s3_client The S3 client used for + # fetching the location of the bucket so that a regional S3 Control + # client can be created. Defaults to the S3 client from the access + # grants plugin. # @option options [String] :privilege ('Default') The privilege to use # when requesting credentials. (see: {Aws::S3Control::Client#get_data_access}) # @option options [Boolean] :fallback (false) When true, if access is @@ -25,7 +32,8 @@ class AccessGrantsCredentialsProvider # @option options [Callable] :before_refresh Proc called before # credentials are refreshed. def initialize(options = {}) - @s3_control_client = options.delete(:s3_control_client) + @s3_control_options = options.delete(:s3_control_client_options) || {} + @s3_client = options.delete(:s3_client) @sts_client = options.delete(:sts_client) @fallback = options.delete(:fallback) || false @caching = options.delete(:caching) || true @@ -34,6 +42,8 @@ def initialize(options = {}) @credentials_cache = ACCESS_GRANTS_CREDENTIALS_CACHE @account_id_cache = ACCESS_GRANTS_ACCOUNT_ID_CACHE + @bucket_region_cache = BUCKET_REGIONS # shared cache with s3_signer + @s3_control_client_cache = {} end def access_grants_credentials_for(options = {}) @@ -42,58 +52,56 @@ def access_grants_credentials_for(options = {}) options[:key], options[:prefix] ) + credentials = s3_client.config.credentials.credentials # resolves if @caching - cached_credentials_for( - target, - options[:credentials], - options[:permission] - ) + cached_credentials_for(target, options[:permission], credentials) else - new_credentials_for( - target, - options[:credentials], - options[:permission] - ) + new_credentials_for(target, options[:permission], credentials) end rescue Aws::S3Control::Errors::AccessDenied raise unless @fallback warn 'Access denied for S3 Access Grants. Falling back to ' \ 'configured credentials.' - options[:credentials] + s3_client.config.credentials end - attr_accessor :s3_credentials - attr_accessor :s3_region + attr_accessor :s3_client private - def cached_credentials_for(target, credentials, permission) - key = credentials_cache_key(target, credentials, permission) + def s3_control_client(bucket_region) + @s3_control_client_cache[bucket_region] ||= begin + credentials = s3_client.config.credentials + config = { credentials: credentials }.merge(@s3_control_options) + Aws::S3Control::Client.new(config.merge(region: bucket_region)) + end + end + + def cached_credentials_for(target, permission, credentials) + key = credentials_cache_key(target, permission, credentials) if @credentials_cache.key?(key) @credentials_cache[key] else @credentials_cache[key] = new_credentials_for( target, - credentials, - permission + permission, + credentials ) end end - def new_credentials_for(target, credentials, permission) - @s3_control_client ||= Aws::S3Control::Client.new( - credentials: s3_credentials, - region: s3_region - ) + def new_credentials_for(target, permission, credentials) + bucket_region = bucket_region_for_access_grants(target) + client = s3_control_client(bucket_region) AccessGrantsCredentials.new( target: target, account_id: account_id_for_access_grants(target, credentials), permission: permission, - client: @s3_control_client, + client: client, **@options ) end @@ -107,33 +115,59 @@ def account_id_for_access_grants(target, credentials) end def cached_account_id_for(target, credentials) - key = account_id_cache_key(target) + bucket = bucket_name_from(target) - if @account_id_cache.key?(key) - @account_id_cache[key] + if @account_id_cache.key?(bucket) + @account_id_cache[bucket] else - @account_id_cache[key] = new_account_id_for(target, credentials) + @account_id_cache[bucket] = new_account_id_for(target, credentials) end end # returns the account id associated with the access grants instance def new_account_id_for(target, credentials) - resp = @s3_control_client.get_access_grants_instance_for_prefix( + bucket_region = bucket_region_for_access_grants(target) + s3_control_client = s3_control_client(bucket_region) + resp = s3_control_client.get_access_grants_instance_for_prefix( s3_prefix: target, - account_id: account_id_for_credentials(credentials), + account_id: account_id_for_credentials(bucket_region, credentials) ) ARNParser.parse(resp.access_grants_instance_arn).account_id end + def bucket_region_for_access_grants(target) + bucket = bucket_name_from(target) + if @caching + cached_bucket_region_for(bucket) + else + new_bucket_region_for(bucket) + end + end + + def cached_bucket_region_for(bucket) + if @bucket_region_cache.key?(bucket) + @bucket_region_cache[bucket] + else + @bucket_region_cache[bucket] = new_bucket_region_for(bucket) + end + end + + def new_bucket_region_for(bucket) + @s3_client.head_bucket(bucket: bucket).bucket_region + rescue Aws::S3::Errors::Http301Error => e + e.data.region + end + # returns the account id for the configured credentials - def account_id_for_credentials(credentials) + def account_id_for_credentials(region, credentials) + # use resolved credentials to check for account id if credentials.respond_to?(:account_id) && credentials.account_id && !credentials.account_id.empty? credentials.account_id else @sts_client ||= Aws::STS::Client.new( - credentials: s3_credentials, - region: s3_region + credentials: s3_client.config.credentials, + region: region ) @sts_client.get_caller_identity.account end @@ -150,13 +184,13 @@ def target_prefix(bucket, key, prefix) end # TODO - should check multiple targets and permissions in a smart way - def credentials_cache_key(target, credentials, permission) + def credentials_cache_key(target, permission, credentials) "#{credentials.access_key_id}-#{credentials.secret_access_key}" \ "-#{permission}-#{target}" end - # uses bucket name from target prefix - def account_id_cache_key(target) + # extracts bucket name from target prefix + def bucket_name_from(target) URI(target).host end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb index b0c1b2581fc..8ca06fb9919 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb @@ -59,6 +59,14 @@ def []=(bucket_name, region_name) end end + # @param [String] key + # @return [Boolean] + def key?(key) + @mutex.synchronize do + @regions.key?(key) + end + end + # @api private def clear @mutex.synchronize { @regions = {} } diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/errors.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/errors.rb index 0b0b4aae2bd..8cd741555ce 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/errors.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/errors.rb @@ -3,8 +3,8 @@ module Aws module S3 module Errors - # Hijack PermanentRedirect dynamic error to also include endpoint - # and bucket. + # Hijack PermanentRedirect dynamic error to include the bucket, region, + # and endpoint. class PermanentRedirect < ServiceError # @param [Seahorse::Client::RequestContext] context # @param [String] message @@ -22,6 +22,19 @@ def initialize(context, message, _data = Aws::EmptyStructure.new) super(context, message, data) end end + + # Hijack PermanentRedirect (HeadBucket case - no body) dynamic error to + # include the region. + class Http301Error < ServiceError + # @param [Seahorse::Client::RequestContext] context + # @param [String] message + # @param [Aws::S3::Types::PermanentRedirect] _data + def initialize(context, message, _data = Aws::EmptyStructure.new) + data = Aws::S3::Types::PermanentRedirect.new(message: message) + data.region = context.http_response.headers['x-amz-bucket-region'] + super(context, message, data) + end + end end end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb index 62d4831122e..bece9699826 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb @@ -55,7 +55,6 @@ def call(context) bucket: params[:bucket], key: params[:key], prefix: params[:prefix], - credentials: context.config.credentials.credentials, permission: permission ) context[:sigv4_credentials] = credentials # Sign will use this @@ -83,8 +82,7 @@ def add_handlers(handlers,config) def after_initialize(client) provider = client.config.access_grants_credentials_provider - provider.s3_credentials = client.config.credentials - provider.s3_region = client.config.region + provider.s3_client = client unless provider.s3_client end end end From d2ec4fc8e0760dc24d8e32f38cf9354062edac28 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Tue, 6 Feb 2024 18:17:59 -0500 Subject: [PATCH 05/19] Broad cache searching --- .../aws-sdk-s3/access_grants_credentials.rb | 15 +++-- .../access_grants_credentials_provider.rb | 64 +++++++++++++++---- 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb index 47115817a7d..ced42c161c1 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb @@ -24,16 +24,21 @@ def initialize(options = {}) # @return [S3Control::Client] attr_reader :client + # @return [String] + attr_reader :matched_grant_target + private def refresh - c = @client.get_data_access(@get_data_access_params).credentials + c = @client.get_data_access(@get_data_access_params) + credentials = c.credentials + @matched_grant_target = c.matched_grant_target @credentials = Credentials.new( - c.access_key_id, - c.secret_access_key, - c.session_token + credentials.access_key_id, + credentials.secret_access_key, + credentials.session_token ) - @expiration = c.expiration + @expiration = credentials.expiration end class << self diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb index acb3e4b5215..d02eb78208e 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb @@ -75,22 +75,61 @@ def s3_control_client(bucket_region) @s3_control_client_cache[bucket_region] ||= begin credentials = s3_client.config.credentials config = { credentials: credentials }.merge(@s3_control_options) - Aws::S3Control::Client.new(config.merge(region: bucket_region)) + Aws::S3Control::Client.new(config.merge( + region: bucket_region, + use_fips_endpoint: s3_client.config.use_fips_endpoint, + use_dualstack_endpoint: s3_client.config.use_dualstack_endpoint + )) end end def cached_credentials_for(target, permission, credentials) - key = credentials_cache_key(target, permission, credentials) + cached_creds = broad_search_credentials_cache_prefix(target, permission, credentials) + return cached_creds if cached_creds - if @credentials_cache.key?(key) - @credentials_cache[key] - else - @credentials_cache[key] = new_credentials_for( - target, - permission, - credentials - ) + if %w[READ WRITE].include?(permission) + cached_creds = broad_search_credentials_cache_prefix(target, 'READWRITE', credentials) + return cached_creds if cached_creds + end + + cached_creds = broad_search_credentials_cache_characters(target, permission, credentials) + return cached_creds if cached_creds + + if %w[READ WRITE].include?(permission) + cached_creds = broad_search_credentials_cache_characters(target, 'READWRITE', credentials) + return cached_creds if cached_creds + end + + creds = new_credentials_for(target, permission, credentials) + if creds.matched_grant_target.end_with?('*') + # remove /* from the end of the target + key = credentials_cache_key(creds.matched_grant_target[0...-2], permission, credentials) + @credentials_cache[key] = creds + end + + creds + end + + def broad_search_credentials_cache_prefix(target, permission, credentials) + prefix = target + while prefix != 's3:' + key = credentials_cache_key(prefix, permission, credentials) + return @credentials_cache[key] if @credentials_cache.key?(key) + + prefix = prefix.split('/', -1)[0..-2].join('/') + end + nil + end + + def broad_search_credentials_cache_characters(target, permission, credentials) + prefix = target + while prefix != 's3://' + key = credentials_cache_key("#{prefix}*", permission, credentials) + return @credentials_cache[key] if @credentials_cache.key?(key) + + prefix = prefix[0..-2] end + nil end def new_credentials_for(target, permission, credentials) @@ -167,7 +206,9 @@ def account_id_for_credentials(region, credentials) else @sts_client ||= Aws::STS::Client.new( credentials: s3_client.config.credentials, - region: region + region: region, + use_fips_endpoint: s3_client.config.use_fips_endpoint, + use_dualstack_endpoint: s3_client.config.use_dualstack_endpoint ) @sts_client.get_caller_identity.account end @@ -183,7 +224,6 @@ def target_prefix(bucket, key, prefix) end end - # TODO - should check multiple targets and permissions in a smart way def credentials_cache_key(target, permission, credentials) "#{credentials.access_key_id}-#{credentials.secret_access_key}" \ "-#{permission}-#{target}" From 5798d7c7d670fb9c976817b2a2f9b2dcc3ca1958 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Tue, 13 Feb 2024 15:24:41 -0500 Subject: [PATCH 06/19] Bump minimum s3 version --- build_tools/services.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/services.rb b/build_tools/services.rb index d9cbc6ebcae..d1690f20fd4 100644 --- a/build_tools/services.rb +++ b/build_tools/services.rb @@ -13,7 +13,7 @@ class ServiceEnumerator MINIMUM_CORE_VERSION = "3.191.0" # Minimum `aws-sdk-core` version for new S3 gem builds - MINIMUM_CORE_VERSION_S3 = "3.191.0" + MINIMUM_CORE_VERSION_S3 = "3.192.0" EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration" From 87e981af8b335250c14e2607a66fe0b216d11007 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Tue, 13 Feb 2024 15:31:43 -0500 Subject: [PATCH 07/19] Fix benchmark --- tasks/benchmark.rake | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/benchmark.rake b/tasks/benchmark.rake index 9f07266485e..8aad26eb015 100644 --- a/tasks/benchmark.rake +++ b/tasks/benchmark.rake @@ -9,6 +9,7 @@ end desc 'Upload/archive the benchmark report' task 'benchmark:archive' do $:.unshift("#{$GEMS_DIR}/aws-sdk-s3/lib") + $:.unshift("#{$GEMS_DIR}/aws-sdk-s3control/lib") $:.unshift("#{$GEMS_DIR}/aws-sdk-kms/lib") require 'aws-sdk-s3' require 'securerandom' From c0b39fe3d4f9dc432c85b2fb5cd9b5f5d6842c19 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Tue, 13 Feb 2024 16:03:44 -0500 Subject: [PATCH 08/19] Add access grants credentials spec and rbs type for provider --- .../lib/aws-sdk-s3/plugins/access_grants.rb | 1 + .../spec/access_grants_credentials_spec.rb | 139 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb index bece9699826..3331e50db8f 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb @@ -17,6 +17,7 @@ class AccessGrants < Seahorse::Client::Plugin option(:access_grants_credentials_provider, doc_type: 'Aws::S3::AccessGrantsCredentialsProvider', + rbs_type: 'untyped', docstring: <<-DOCS) do |_cfg| When `s3_access_grants` is `true`, this option can be used to provide additional options to the credentials provider, including a privilege diff --git a/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb b/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb new file mode 100644 index 00000000000..8e41e609b04 --- /dev/null +++ b/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' + +module Aws + module S3 + describe AccessGrantsCredentials do + + let(:client) do + S3Control::Client.new(region: 'us-east-1', stub_responses: true) + end + + let(:in_five_minutes) { Time.now + 60 * 5 } + + let(:expiration) { in_five_minutes } + + let(:credentials) do + double('credentials', + access_key_id: 'akid', + secret_access_key: 'secret', + session_token: 'session', + expiration: expiration, + ) + end + + let(:matched_grant_target) { 's3://bucket/*' } + + let(:resp) do + double('client-resp', + credentials: credentials, + matched_grant_target: matched_grant_target + ) + end + + before(:each) do + allow(client).to receive(:get_data_access).and_return(resp) + end + + it 'gets data access using the client and options' do + expect(client).to receive(:get_data_access).with({ + account_id: '12345678910', + target: 's3://bucket', + permission: 'READWRITE' + }).and_return(resp) + + AccessGrantsCredentials.new( + client: client, + account_id: '12345678910', + target: 's3://bucket', + permission: 'READWRITE' + ) + end + + it 'extracts credentials from the data access response' do + c = AccessGrantsCredentials.new( + client: client, + account_id: '12345678910', + target: 's3://bucket', + permission: 'READWRITE' + ) + + expect(c).to be_set + expect(c.credentials.access_key_id).to eq('akid') + expect(c.credentials.secret_access_key).to eq('secret') + expect(c.credentials.session_token).to eq('session') + expect(c.expiration).to eq(in_five_minutes) + end + + it 'provides the matched grant target' do + c = AccessGrantsCredentials.new( + client: client, + account_id: '12345678910', + target: 's3://bucket', + permission: 'READWRITE' + ) + + expect(c.matched_grant_target).to eq('s3://bucket/*') + end + + it 'refreshes asynchronously' do + # expiration 9.5 minutes out, within the async exp time window + time = Time.now + 60 * 9.5 + allow(credentials).to receive(:expiration).and_return(time) + expect(client).to receive(:get_data_access).at_least(2).times + expect(Thread).to receive(:new).and_yield + c = AccessGrantsCredentials.new( + client: client, + account_id: '12345678910', + target: 's3://bucket', + permission: 'READWRITE' + ) + c.credentials + end + + it 'refreshes credentials automatically when they are near expiration' do + allow(credentials).to receive(:expiration).and_return(Time.now) + expect(client).to receive(:get_data_access).exactly(4).times + c = AccessGrantsCredentials.new( + client: client, + account_id: '12345678910', + target: 's3://bucket', + permission: 'READWRITE' + ) + c.credentials + c.credentials + c.credentials + end + + it 'expiration is a Time' do + c = AccessGrantsCredentials.new( + client: client, + account_id: '12345678910', + target: 's3://bucket', + permission: 'READWRITE' + ) + expect(c.expiration).to be_a(Time) + end + + it 'calls before_refresh with self' do + before_refresh_called = false + before_refresh = proc do |cred_provider| + before_refresh_called = true + expect(cred_provider).to be_instance_of(AccessGrantsCredentials) + end + + AccessGrantsCredentials.new( + client: client, + account_id: '12345678910', + target: 's3://bucket', + permission: 'READWRITE', + before_refresh: before_refresh + ) + + expect(before_refresh_called).to be(true) + end + end + end +end + From 6ec20bead1091c0092999eb91518c0440d470c66 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Wed, 14 Feb 2024 16:38:51 -0500 Subject: [PATCH 09/19] Credentials provider spec --- .../access_grants_credentials_provider.rb | 19 +- ...access_grants_credentials_provider_spec.rb | 347 ++++++++++++++++++ gems/aws-sdk-s3/spec/client_spec.rb | 19 +- 3 files changed, 372 insertions(+), 13 deletions(-) create mode 100644 gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb index d02eb78208e..911ee6fe4fe 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb @@ -36,14 +36,13 @@ def initialize(options = {}) @s3_client = options.delete(:s3_client) @sts_client = options.delete(:sts_client) @fallback = options.delete(:fallback) || false - @caching = options.delete(:caching) || true - @options = options + @caching = options.delete(:caching) != false + @s3_control_clients = {} + @bucket_region_cache = BUCKET_REGIONS # shared cache with s3_signer return unless @caching @credentials_cache = ACCESS_GRANTS_CREDENTIALS_CACHE @account_id_cache = ACCESS_GRANTS_ACCOUNT_ID_CACHE - @bucket_region_cache = BUCKET_REGIONS # shared cache with s3_signer - @s3_control_client_cache = {} end def access_grants_credentials_for(options = {}) @@ -72,7 +71,7 @@ def access_grants_credentials_for(options = {}) private def s3_control_client(bucket_region) - @s3_control_client_cache[bucket_region] ||= begin + @s3_control_clients[bucket_region] ||= begin credentials = s3_client.config.credentials config = { credentials: credentials }.merge(@s3_control_options) Aws::S3Control::Client.new(config.merge( @@ -140,8 +139,7 @@ def new_credentials_for(target, permission, credentials) target: target, account_id: account_id_for_access_grants(target, credentials), permission: permission, - client: client, - **@options + client: client ) end @@ -176,11 +174,8 @@ def new_account_id_for(target, credentials) def bucket_region_for_access_grants(target) bucket = bucket_name_from(target) - if @caching - cached_bucket_region_for(bucket) - else - new_bucket_region_for(bucket) - end + # regardless of caching option, bucket region cache is always shared + cached_bucket_region_for(bucket) end def cached_bucket_region_for(bucket) diff --git a/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb b/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb new file mode 100644 index 00000000000..69aa81cf184 --- /dev/null +++ b/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb @@ -0,0 +1,347 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' + +module Aws + module S3 + describe AccessGrantsCredentialsProvider do + let(:s3_client) { Aws::S3::Client.new(stub_responses: true) } + let(:sts_client) { Aws::STS::Client.new(stub_responses: true) } + let(:s3_control_client) { Aws::S3Control::Client.new(stub_responses: true) } + + let(:bucket) { 'bucket' } + let(:key) { 'key' } + let(:prefix) { 'prefix' } + let(:permission) { 'READ' } + let(:matched_grant_target) { 's3://bucket/key/*' } + + let(:access_grants_credentials) do + double( + 'access_grants_credentials', + credentials: Aws::Credentials.new( + 'access-grants-akid', + 'access-grants-secret', + 'access-grants-token' + ), + matched_grant_target: matched_grant_target, + expiration: Time.now + 3600 + ) + end + + subject do + AccessGrantsCredentialsProvider.new( + s3_client: s3_client, + sts_client: sts_client, + bucket: bucket, + key: key, + permission: permission + ) + end + + before do + allow(Aws::S3Control::Client).to receive(:new).and_return(s3_control_client) + end + + after do + ACCESS_GRANTS_CREDENTIALS_CACHE.clear + ACCESS_GRANTS_ACCOUNT_ID_CACHE.clear + BUCKET_REGIONS.clear + end + + describe '#access_grants_credentials_for' do + before do + s3_client.stub_responses(:head_bucket, bucket_region: 'us-west-2') + sts_client.stub_responses(:get_caller_identity, account: '123456789012') + s3_control_client.stub_responses( + :get_access_grants_instance_for_prefix, + access_grants_instance_arn: 'arn:aws:s3:us-west-2:120987654321:access-grants' + ) + end + + it 'returns credentials' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/key', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).and_return(access_grants_credentials) + + creds = subject.access_grants_credentials_for( + bucket: bucket, + key: key, + permission: permission + ).credentials + expect(creds.access_key_id).to eq('access-grants-akid') + expect(creds.secret_access_key).to eq('access-grants-secret') + expect(creds.session_token).to eq('access-grants-token') + end + + it 'raises when access denied' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/key', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).and_raise(Aws::S3Control::Errors::AccessDenied.new(nil, 'AccessDenied')) + + expect do + subject.access_grants_credentials_for( + bucket: bucket, + key: key, + permission: permission + ) + end.to raise_error(Aws::S3Control::Errors::AccessDenied) + end + + it 'caches credentials' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/key', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).and_return(access_grants_credentials) + expect(s3_client).to receive(:head_bucket).once.and_call_original + expect(sts_client).to receive(:get_caller_identity) + .once.and_call_original + expect(s3_control_client) + .to receive(:get_access_grants_instance_for_prefix) + .once.and_call_original + + creds = subject.access_grants_credentials_for( + bucket: bucket, + key: key, + permission: permission + ) + expect(creds).to eq( + subject.access_grants_credentials_for( + bucket: bucket, + key: key, + permission: permission + ) + ) + end + + context 'fallback enabled' do + subject do + AccessGrantsCredentialsProvider.new( + s3_client: s3_client, + sts_client: sts_client, + bucket: bucket, + key: key, + permission: permission, + fallback: true + ) + end + + it 'falls back to original s3 credentials' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/key', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).and_raise(Aws::S3Control::Errors::AccessDenied.new(nil, 'AccessDenied')) + expect_any_instance_of(AccessGrantsCredentialsProvider) + .to receive(:warn).with(/Access Grants/) + + creds = subject.access_grants_credentials_for( + bucket: bucket, + key: key, + permission: permission + ) + expect(creds).to eq(s3_client.config.credentials) + end + end + + context 'broad cache search' do + let(:matched_grant_target) { 's3://bucket/folder/*' } + + it 'searches prefix' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/folder/key', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).and_return(access_grants_credentials) + subject.access_grants_credentials_for( + bucket: bucket, + key: 'folder/key', + permission: permission + ) + expect(AccessGrantsCredentials).not_to receive(:new) + subject.access_grants_credentials_for( + bucket: bucket, + key: 'folder', + permission: permission + ) + end + + it 'searches prefix with READWRITE permission' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/folder/key', + account_id: '120987654321', + permission: 'READWRITE', + client: s3_control_client + ).and_return(access_grants_credentials) + subject.access_grants_credentials_for( + bucket: bucket, + key: 'folder/key', + permission: 'READWRITE' + ) + expect(AccessGrantsCredentials).not_to receive(:new) + subject.access_grants_credentials_for( + bucket: bucket, + key: 'folder', + permission: permission + ) + end + + it 'searches characters' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/folder/key', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).and_return(access_grants_credentials) + subject.access_grants_credentials_for( + bucket: bucket, + key: 'folder/key', + permission: permission + ) + expect(AccessGrantsCredentials).not_to receive(:new) + subject.access_grants_credentials_for( + bucket: bucket, + key: 'folder/k', + permission: permission + ) + end + + it 'searches characters with READWRITE permission' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/folder/key', + account_id: '120987654321', + permission: 'READWRITE', + client: s3_control_client + ).and_return(access_grants_credentials) + subject.access_grants_credentials_for( + bucket: bucket, + key: 'folder/key', + permission: 'READWRITE' + ) + expect(AccessGrantsCredentials).not_to receive(:new) + subject.access_grants_credentials_for( + bucket: bucket, + key: 'folder/k', + permission: permission + ) + end + end + + context 'caching disabled' do + subject do + AccessGrantsCredentialsProvider.new( + s3_client: s3_client, + sts_client: sts_client, + bucket: bucket, + key: key, + permission: permission, + caching: false + ) + end + + it 'does not cache credentials' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/key', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).twice.and_return(access_grants_credentials) + expect(s3_client).to receive(:head_bucket).once.and_call_original + expect(sts_client).to receive(:get_caller_identity) + .twice.and_call_original + expect(s3_control_client) + .to receive(:get_access_grants_instance_for_prefix) + .twice.and_call_original + + subject.access_grants_credentials_for( + bucket: bucket, + key: key, + permission: permission + ) + subject.access_grants_credentials_for( + bucket: bucket, + key: key, + permission: permission + ) + end + end + + context 'prefix' do + subject do + AccessGrantsCredentialsProvider.new( + s3_client: s3_client, + sts_client: sts_client, + bucket: bucket, + prefix: prefix, + permission: permission + ) + end + + it 'returns credentials' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/prefix', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).and_return(access_grants_credentials) + + creds = subject.access_grants_credentials_for( + bucket: bucket, + prefix: prefix, + permission: permission + ).credentials + expect(creds.access_key_id).to eq('access-grants-akid') + expect(creds.secret_access_key).to eq('access-grants-secret') + expect(creds.session_token).to eq('access-grants-token') + end + end + + context 'no key or prefix' do + subject do + AccessGrantsCredentialsProvider.new( + s3_client: s3_client, + sts_client: sts_client, + bucket: bucket, + permission: permission + ) + end + + it 'returns credentials' do + expect(AccessGrantsCredentials).to receive(:new).with( + target: 's3://bucket/*', + account_id: '120987654321', + permission: 'READ', + client: s3_control_client + ).and_return(access_grants_credentials) + + creds = subject.access_grants_credentials_for( + bucket: bucket, + permission: permission + ).credentials + expect(creds.access_key_id).to eq('access-grants-akid') + expect(creds.secret_access_key).to eq('access-grants-secret') + expect(creds.session_token).to eq('access-grants-token') + end + end + end + + describe '#s3_client' do + it 'can assign and return the s3_client' do + expect(subject.s3_client).to eq(s3_client) + new_client = double('client') + subject.s3_client = new_client + expect(subject.s3_client).to eq(new_client) + end + end + end + end +end diff --git a/gems/aws-sdk-s3/spec/client_spec.rb b/gems/aws-sdk-s3/spec/client_spec.rb index e3b4b71ef7e..59068229153 100644 --- a/gems/aws-sdk-s3/spec/client_spec.rb +++ b/gems/aws-sdk-s3/spec/client_spec.rb @@ -106,7 +106,7 @@ module S3 end describe 'permanent redirect error' do - it 'includes endpoint and bucket in PermanentRedirect' do + it 'includes endpoint, bucket, and region in PermanentRedirect' do client = Client.new(stub_responses: true) client.handle(step: :send) do |context| context.http_response.signal_done( @@ -132,6 +132,23 @@ module S3 expect(error.data.region).to eq('us-peccy-1') end end + + it 'handles PermanentRedirect with no body' do + client = Client.new(stub_responses: true) + client.handle(step: :send) do |context| + context.http_response.signal_done( + status_code: 301, + headers: { 'x-amz-bucket-region' => 'us-peccy-1' }, + body: '' + ) + Seahorse::Client::Response.new(context: context) + end + expect do + client.head_bucket(bucket: 'bucket') + end.to raise_error(Errors::Http301Error) do |error| + expect(error.data.region).to eq('us-peccy-1') + end + end end describe 'unlinked tempfiles' do From 698636730efc57fbe577f405c2a6885c08f7ebc7 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Thu, 15 Feb 2024 14:14:16 -0500 Subject: [PATCH 10/19] Revert codegen changes and add specs for the plugin --- gems/aws-sdk-s3/aws-sdk-s3.gemspec | 2 +- gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb | 8 -- .../lib/aws-sdk-s3/plugins/access_grants.rb | 9 +-- .../spec/plugins/access_grants_spec.rb | 81 +++++++++++++++++++ .../spec/plugins/express_session_auth_spec.rb | 11 +++ 5 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb diff --git a/gems/aws-sdk-s3/aws-sdk-s3.gemspec b/gems/aws-sdk-s3/aws-sdk-s3.gemspec index 415daa16c7d..0e67d8c520b 100644 --- a/gems/aws-sdk-s3/aws-sdk-s3.gemspec +++ b/gems/aws-sdk-s3/aws-sdk-s3.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency('aws-sdk-kms', '~> 1') spec.add_dependency('aws-sdk-s3control', '~> 1.73') spec.add_dependency('aws-sigv4', '~> 1.8') - spec.add_dependency('aws-sdk-core', '~> 3', '>= 3.191.0') + spec.add_dependency('aws-sdk-core', '~> 3', '>= 3.192.0') spec.required_ruby_version = '>= 2.5' end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb index 3121f7ca8ae..c2283a7e927 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb @@ -34,7 +34,6 @@ require 'aws-sdk-core/plugins/sign.rb' require 'aws-sdk-core/plugins/protocols/rest_xml.rb' require 'aws-sdk-s3/plugins/accelerate.rb' -require 'aws-sdk-s3/plugins/access_grants.rb' require 'aws-sdk-s3/plugins/arn.rb' require 'aws-sdk-s3/plugins/bucket_dns.rb' require 'aws-sdk-s3/plugins/bucket_name_restrictions.rb' @@ -105,7 +104,6 @@ class Client < Seahorse::Client::Base add_plugin(Aws::Plugins::Sign) add_plugin(Aws::Plugins::Protocols::RestXml) add_plugin(Aws::S3::Plugins::Accelerate) - add_plugin(Aws::S3::Plugins::AccessGrants) add_plugin(Aws::S3::Plugins::ARN) add_plugin(Aws::S3::Plugins::BucketDns) add_plugin(Aws::S3::Plugins::BucketNameRestrictions) @@ -368,12 +366,6 @@ class Client < Seahorse::Client::Base # in the future. # # - # @option options [Boolean] :s3_access_grants (false) - # TODO - # - # @option options [Aws::S3::AccessGrantsCredentialsProvider] :s3_access_grants_credentials_provider - # TODO - # # @option options [Boolean] :s3_disable_multiregion_access_points (false) # When set to `false` this will option will raise errors when multi-region # access point ARNs are used. Multi-region access points can potentially diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb index 3331e50db8f..5028bb666eb 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb @@ -6,7 +6,7 @@ module Plugins # @api private class AccessGrants < Seahorse::Client::Plugin option( - :s3_access_grants, + :access_grants, default: false, doc_type: 'Boolean', docstring: <<-DOCS) @@ -19,7 +19,7 @@ class AccessGrants < Seahorse::Client::Plugin doc_type: 'Aws::S3::AccessGrantsCredentialsProvider', rbs_type: 'untyped', docstring: <<-DOCS) do |_cfg| -When `s3_access_grants` is `true`, this option can be used to provide +When `access_grants` is `true`, this option can be used to provide additional options to the credentials provider, including a privilege setting, caching, and fallback behavior. DOCS @@ -72,13 +72,12 @@ def access_grants_operation?(context) end def s3_express_endpoint?(context) - props = context[:endpoint_properties] - props['backend'] == 'S3Express' + context[:endpoint_properties]['backend'] == 'S3Express' end end def add_handlers(handlers,config) - handlers.add(Handler) if config.s3_access_grants + handlers.add(Handler) if config.access_grants end def after_initialize(client) diff --git a/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb b/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb new file mode 100644 index 00000000000..434511b8890 --- /dev/null +++ b/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb @@ -0,0 +1,81 @@ +require_relative '../spec_helper' + +module Aws + module S3 + describe Client do + describe 'access_grants' do + it 'is disabled by default' do + client = Aws::S3::Client.new( + stub_responses: true, + region: 'us-west-2' + ) + expect(client.config.access_grants).to be false + end + end + + describe 'access_grants_credentials_provider' do + it 'is configured to use the default provider by default' do + client = Aws::S3::Client.new( + stub_responses: true, + region: 'us-west-2' + ) + expect(client.config.access_grants_credentials_provider) + .to be_a(Aws::S3::AccessGrantsCredentialsProvider) + end + + it 'can use a configured provider' do + provider = Aws::S3::AccessGrantsCredentialsProvider.new + client = Aws::S3::Client.new( + stub_responses: true, + region: 'us-west-2', + access_grants_credentials_provider: provider + ) + expect(client.config.access_grants_credentials_provider) + .to be(provider) + end + end + + it 'sets the s3_client as the client to get data access' do + client = Aws::S3::Client.new( + stub_responses: true, + region: 'us-west-2' + ) + provider = client.config.access_grants_credentials_provider + expect(provider.s3_client).to eq(client) + end + + let(:client) do + Aws::S3::Client.new( + stub_responses: true, + access_grants: true, + region: 'us-east-1' + ) + end + + it 'is skipped for s3 express endpoints' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.head_object(bucket: 'bucket--use1-az2--x-s3', key: 'key') + end + + it 'is skipped for non-bucket operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.list_buckets + end + + it 'is skipped for non-permissioned operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.list_objects(bucket: 'bucket') + end + + it 'is called for permissioned bucket operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .to receive(:access_grants_credentials_for) + .with(bucket: 'bucket', key: 'key', permission: 'READ', prefix: nil) + client.head_object(bucket: 'bucket', key: 'key') + end + end + end +end diff --git a/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb b/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb index f25e53adeb7..43f0738849f 100644 --- a/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb +++ b/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb @@ -54,6 +54,17 @@ module S3 expect(client.config.express_credentials_provider) .to be_a(Aws::S3::ExpressCredentialsProvider) end + + it 'can use a configured provider' do + provider = Aws::S3::ExpressCredentialsProvider.new + client = Aws::S3::Client.new( + stub_responses: true, + region: 'us-west-2', + express_credentials_provider: provider + ) + expect(client.config.express_credentials_provider) + .to be(provider) + end end it 'sets the client as the client to create sessions' do From 5ca578ff7171a040c8a1a4e5a789346672e19ab9 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Sun, 28 Apr 2024 23:38:55 -0400 Subject: [PATCH 11/19] Remove s3control as a dependency to s3 and use loading approach --- gems/aws-sdk-s3/CHANGELOG.md | 4 +--- gems/aws-sdk-s3/aws-sdk-s3.gemspec | 1 - gems/aws-sdk-s3/features/env.rb | 1 - gems/aws-sdk-s3/lib/aws-sdk-s3.rb | 1 - gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb | 12 ++++++++++++ .../lib/aws-sdk-s3/plugins/access_grants.rb | 5 +++++ gems/aws-sdk-s3/sig/client.rbs | 2 ++ gems/aws-sdk-s3/sig/resource.rbs | 2 ++ gems/aws-sdk-s3/spec/spec_helper.rb | 1 - services.json | 1 - 10 files changed, 22 insertions(+), 8 deletions(-) diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index 6c3e7cff2a0..db5ca9fa66e 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,9 +1,7 @@ Unreleased Changes ------------------ -* Feature - Support S3 Access Grants authentication. Access Grants can be enabled with the `s3_access_grants` option, and custom options can be passed into the `access_grants_credentials_provider` option. - -* Feature - S3 now depends on `aws-sdk-s3control` for Access Grants credential fetching. +* Feature - Support S3 Access Grants authentication. Access Grants can be enabled with the `s3_access_grants` option, and custom options can be passed into the `access_grants_credentials_provider` option. This feature is enabled if `aws-sdk-s3control` is installed. 1.148.0 (2024-04-25) ------------------ diff --git a/gems/aws-sdk-s3/aws-sdk-s3.gemspec b/gems/aws-sdk-s3/aws-sdk-s3.gemspec index 58e35c5e236..a6a7b30b156 100644 --- a/gems/aws-sdk-s3/aws-sdk-s3.gemspec +++ b/gems/aws-sdk-s3/aws-sdk-s3.gemspec @@ -26,7 +26,6 @@ Gem::Specification.new do |spec| } spec.add_dependency('aws-sdk-kms', '~> 1') - spec.add_dependency('aws-sdk-s3control', '~> 1.73') spec.add_dependency('aws-sigv4', '~> 1.8') spec.add_dependency('aws-sdk-core', '~> 3', '>= 3.194.0') diff --git a/gems/aws-sdk-s3/features/env.rb b/gems/aws-sdk-s3/features/env.rb index 138b4f3cd53..2d204f85107 100644 --- a/gems/aws-sdk-s3/features/env.rb +++ b/gems/aws-sdk-s3/features/env.rb @@ -10,7 +10,6 @@ $:.unshift(File.expand_path('../../lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-core/features', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-kms/lib', __FILE__)) -$:.unshift(File.expand_path('../../../aws-sdk-s3control/lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sigv4/lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-core/lib', __FILE__)) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3.rb index 90fb562be59..285d7400c1c 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3.rb @@ -9,7 +9,6 @@ require 'aws-sdk-kms' -require 'aws-sdk-s3control' require 'aws-sigv4' require 'aws-sdk-core' diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb index 10dfa200b2f..4552e114189 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb @@ -35,6 +35,7 @@ require 'aws-sdk-core/plugins/sign.rb' require 'aws-sdk-core/plugins/protocols/rest_xml.rb' require 'aws-sdk-s3/plugins/accelerate.rb' +require 'aws-sdk-s3/plugins/access_grants.rb' require 'aws-sdk-s3/plugins/arn.rb' require 'aws-sdk-s3/plugins/bucket_dns.rb' require 'aws-sdk-s3/plugins/bucket_name_restrictions.rb' @@ -106,6 +107,7 @@ class Client < Seahorse::Client::Base add_plugin(Aws::Plugins::Sign) add_plugin(Aws::Plugins::Protocols::RestXml) add_plugin(Aws::S3::Plugins::Accelerate) + add_plugin(Aws::S3::Plugins::AccessGrants) add_plugin(Aws::S3::Plugins::ARN) add_plugin(Aws::S3::Plugins::BucketDns) add_plugin(Aws::S3::Plugins::BucketNameRestrictions) @@ -186,6 +188,16 @@ class Client < Seahorse::Client::Base # * `~/.aws/credentials` # * `~/.aws/config` # + # @option options [Boolean] :access_grants (false) + # When `true`, the S3 client will use the S3 Access Grants feature to + # authenticate requests. Bucket credentials will be fetched from S3 + # Control using the `get_data_access` API. + # + # @option options [Aws::S3::AccessGrantsCredentialsProvider] :access_grants_credentials_provider + # When `access_grants` is `true`, this option can be used to provide + # additional options to the credentials provider, including a privilege + # setting, caching, and fallback behavior. + # # @option options [String] :access_key_id # # @option options [Boolean] :active_endpoint_cache (false) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb index 5028bb666eb..65d9acab98d 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb @@ -77,10 +77,15 @@ def s3_express_endpoint?(context) end def add_handlers(handlers,config) + require 'aws-sdk-s3control' handlers.add(Handler) if config.access_grants + rescue LoadError + false end def after_initialize(client) + return unless client.config.access_grants + provider = client.config.access_grants_credentials_provider provider.s3_client = client unless provider.s3_client end diff --git a/gems/aws-sdk-s3/sig/client.rbs b/gems/aws-sdk-s3/sig/client.rbs index 1e49bfb0684..d882c6cbea0 100644 --- a/gems/aws-sdk-s3/sig/client.rbs +++ b/gems/aws-sdk-s3/sig/client.rbs @@ -14,6 +14,8 @@ module Aws def self.new: ( ?credentials: untyped, ?region: String, + ?access_grants: bool, + ?access_grants_credentials_provider: untyped, ?access_key_id: String, ?active_endpoint_cache: bool, ?adaptive_retry_wait_to_fill: bool, diff --git a/gems/aws-sdk-s3/sig/resource.rbs b/gems/aws-sdk-s3/sig/resource.rbs index 8aafdfadc13..bc1f663a0ec 100644 --- a/gems/aws-sdk-s3/sig/resource.rbs +++ b/gems/aws-sdk-s3/sig/resource.rbs @@ -14,6 +14,8 @@ module Aws ?client: Client, ?credentials: untyped, ?region: String, + ?access_grants: bool, + ?access_grants_credentials_provider: untyped, ?access_key_id: String, ?active_endpoint_cache: bool, ?adaptive_retry_wait_to_fill: bool, diff --git a/gems/aws-sdk-s3/spec/spec_helper.rb b/gems/aws-sdk-s3/spec/spec_helper.rb index ea53682b8da..ac98b73f01e 100644 --- a/gems/aws-sdk-s3/spec/spec_helper.rb +++ b/gems/aws-sdk-s3/spec/spec_helper.rb @@ -11,7 +11,6 @@ $:.unshift(File.expand_path('../../lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-kms/lib', __FILE__)) -$:.unshift(File.expand_path('../../../aws-sdk-s3control/lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sigv4/lib', __FILE__)) $:.unshift(File.expand_path('../../../aws-sdk-core/lib', __FILE__)) diff --git a/services.json b/services.json index 857e5d1883c..fda7a965b81 100644 --- a/services.json +++ b/services.json @@ -1002,7 +1002,6 @@ "models": "s3/2006-03-01", "dependencies": { "aws-sdk-kms": "~> 1", - "aws-sdk-s3control": "~> 1.73", "aws-sigv4": "~> 1.8" }, "addPlugins": [ From a01ae557c35d9aff143bd493cf7ef8aaaacc23e2 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Mon, 29 Apr 2024 11:07:17 -0400 Subject: [PATCH 12/19] New approach for loading s3control and fix tests --- .../lib/aws-sdk-s3/plugins/access_grants.rb | 31 +++++-- .../spec/plugins/access_grants_spec.rb | 85 +++++++++++++------ 2 files changed, 82 insertions(+), 34 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb index 65d9acab98d..37c3c28b829 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb @@ -5,6 +5,15 @@ module S3 module Plugins # @api private class AccessGrants < Seahorse::Client::Plugin + @s3control = + begin + require 'aws-sdk-s3control' + puts "required s3control" + true + rescue LoadError + false + end + option( :access_grants, default: false, @@ -58,6 +67,7 @@ def call(context) prefix: params[:prefix], permission: permission ) + puts "credentials resolved: #{credentials}" context[:sigv4_credentials] = credentials # Sign will use this end @@ -76,19 +86,28 @@ def s3_express_endpoint?(context) end end - def add_handlers(handlers,config) - require 'aws-sdk-s3control' - handlers.add(Handler) if config.access_grants - rescue LoadError - false + def add_handlers(handlers, config) + return unless AccessGrants.s3control? && config.access_grants + + puts "adding handler" + + handlers.add(Handler) end def after_initialize(client) - return unless client.config.access_grants + return unless AccessGrants.s3control? && client.config.access_grants + + puts "adding s3 client to provider" provider = client.config.access_grants_credentials_provider provider.s3_client = client unless provider.s3_client end + + class << self + def s3control? + @s3control + end + end end end end diff --git a/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb b/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb index 434511b8890..5626267cd78 100644 --- a/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb +++ b/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb @@ -11,6 +11,25 @@ module S3 ) expect(client.config.access_grants).to be false end + + it 'does not set the s3_client if disabled' do + client = Aws::S3::Client.new( + stub_responses: true, + region: 'us-west-2' + ) + provider = client.config.access_grants_credentials_provider + expect(provider.s3_client).to be_nil + end + + it 'sets the s3_client as the client for get_data_access' do + client = Aws::S3::Client.new( + stub_responses: true, + access_grants: true, + region: 'us-west-2' + ) + provider = client.config.access_grants_credentials_provider + expect(provider.s3_client).to eq(client) + end end describe 'access_grants_credentials_provider' do @@ -35,15 +54,6 @@ module S3 end end - it 'sets the s3_client as the client to get data access' do - client = Aws::S3::Client.new( - stub_responses: true, - region: 'us-west-2' - ) - provider = client.config.access_grants_credentials_provider - expect(provider.s3_client).to eq(client) - end - let(:client) do Aws::S3::Client.new( stub_responses: true, @@ -52,29 +62,48 @@ module S3 ) end - it 'is skipped for s3 express endpoints' do - expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) - .not_to receive(:access_grants_credentials_for) - client.head_object(bucket: 'bucket--use1-az2--x-s3', key: 'key') - end + context 's3control loaded' do + before do + allow(Aws::S3::Plugins::AccessGrants) + .to receive(:s3control?).and_return(true) + end - it 'is skipped for non-bucket operations' do - expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) - .not_to receive(:access_grants_credentials_for) - client.list_buckets - end + it 'is skipped for s3 express endpoints' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.head_object(bucket: 'bucket--use1-az2--x-s3', key: 'key') + end + + it 'is skipped for non-bucket operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.list_buckets + end + + it 'is skipped for non-permissioned operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.list_objects(bucket: 'bucket') + end - it 'is skipped for non-permissioned operations' do - expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) - .not_to receive(:access_grants_credentials_for) - client.list_objects(bucket: 'bucket') + it 'is called for permissioned bucket operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .to receive(:access_grants_credentials_for) + .with(bucket: 'bucket', key: 'key', permission: 'READ', prefix: nil) + client.head_object(bucket: 'bucket', key: 'key') + end end - it 'is called for permissioned bucket operations' do - expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) - .to receive(:access_grants_credentials_for) - .with(bucket: 'bucket', key: 'key', permission: 'READ', prefix: nil) - client.head_object(bucket: 'bucket', key: 'key') + context 's3control not loaded' do + before do + allow(Aws::S3::Plugins::AccessGrants) + .to receive(:s3control?).and_return(false) + end + + it 'does not add the handler' do + expect(client.handlers) + .not_to include(Aws::S3::Plugins::AccessGrants::Handler) + end end end end From ff04d10d06e2ebeb7ad42f64d4b77013bfb854ea Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Mon, 29 Apr 2024 11:16:11 -0400 Subject: [PATCH 13/19] Fix tests and remove puts --- .../lib/aws-sdk-s3/plugins/access_grants.rb | 6 -- .../spec/plugins/access_grants_spec.rb | 57 ++++++++++--------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb index 37c3c28b829..85db2efe79a 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/access_grants.rb @@ -8,7 +8,6 @@ class AccessGrants < Seahorse::Client::Plugin @s3control = begin require 'aws-sdk-s3control' - puts "required s3control" true rescue LoadError false @@ -67,7 +66,6 @@ def call(context) prefix: params[:prefix], permission: permission ) - puts "credentials resolved: #{credentials}" context[:sigv4_credentials] = credentials # Sign will use this end @@ -89,16 +87,12 @@ def s3_express_endpoint?(context) def add_handlers(handlers, config) return unless AccessGrants.s3control? && config.access_grants - puts "adding handler" - handlers.add(Handler) end def after_initialize(client) return unless AccessGrants.s3control? && client.config.access_grants - puts "adding s3 client to provider" - provider = client.config.access_grants_credentials_provider provider.s3_client = client unless provider.s3_client end diff --git a/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb b/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb index 5626267cd78..68df8a0dddd 100644 --- a/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb +++ b/gems/aws-sdk-s3/spec/plugins/access_grants_spec.rb @@ -3,6 +3,11 @@ module Aws module S3 describe Client do + before do + allow(Aws::S3::Plugins::AccessGrants) + .to receive(:s3control?).and_return(true) + end + describe 'access_grants' do it 'is disabled by default' do client = Aws::S3::Client.new( @@ -62,36 +67,29 @@ module S3 ) end - context 's3control loaded' do - before do - allow(Aws::S3::Plugins::AccessGrants) - .to receive(:s3control?).and_return(true) - end - - it 'is skipped for s3 express endpoints' do - expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) - .not_to receive(:access_grants_credentials_for) - client.head_object(bucket: 'bucket--use1-az2--x-s3', key: 'key') - end + it 'is skipped for s3 express endpoints' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.head_object(bucket: 'bucket--use1-az2--x-s3', key: 'key') + end - it 'is skipped for non-bucket operations' do - expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) - .not_to receive(:access_grants_credentials_for) - client.list_buckets - end + it 'is skipped for non-bucket operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.list_buckets + end - it 'is skipped for non-permissioned operations' do - expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) - .not_to receive(:access_grants_credentials_for) - client.list_objects(bucket: 'bucket') - end + it 'is skipped for non-permissioned operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .not_to receive(:access_grants_credentials_for) + client.list_objects(bucket: 'bucket') + end - it 'is called for permissioned bucket operations' do - expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) - .to receive(:access_grants_credentials_for) - .with(bucket: 'bucket', key: 'key', permission: 'READ', prefix: nil) - client.head_object(bucket: 'bucket', key: 'key') - end + it 'is called for permissioned bucket operations' do + expect_any_instance_of(Aws::S3::AccessGrantsCredentialsProvider) + .to receive(:access_grants_credentials_for) + .with(bucket: 'bucket', key: 'key', permission: 'READ', prefix: nil) + client.head_object(bucket: 'bucket', key: 'key') end context 's3control not loaded' do @@ -104,6 +102,11 @@ module S3 expect(client.handlers) .not_to include(Aws::S3::Plugins::AccessGrants::Handler) end + + it 'does not set the s3 client' do + provider = client.config.access_grants_credentials_provider + expect(provider.s3_client).to be_nil + end end end end From 6d11c3ce74e22d957d4817b8af1d24beefdee475 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Mon, 29 Apr 2024 11:17:10 -0400 Subject: [PATCH 14/19] Remove s3control from benchmark rake --- tasks/benchmark.rake | 1 - 1 file changed, 1 deletion(-) diff --git a/tasks/benchmark.rake b/tasks/benchmark.rake index 8aad26eb015..9f07266485e 100644 --- a/tasks/benchmark.rake +++ b/tasks/benchmark.rake @@ -9,7 +9,6 @@ end desc 'Upload/archive the benchmark report' task 'benchmark:archive' do $:.unshift("#{$GEMS_DIR}/aws-sdk-s3/lib") - $:.unshift("#{$GEMS_DIR}/aws-sdk-s3control/lib") $:.unshift("#{$GEMS_DIR}/aws-sdk-kms/lib") require 'aws-sdk-s3' require 'securerandom' From 743d60280cf8b7cacfd0abc625a803382c759279 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Mon, 29 Apr 2024 11:59:42 -0400 Subject: [PATCH 15/19] Fix rbs spy test --- .../lib/aws-sdk-s3/access_grants_credentials.rb | 2 +- .../aws-sdk-s3/spec/access_grants_credentials_spec.rb | 2 +- tasks/rbs.rake | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb index ced42c161c1..f6922eaeab3 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials.rb @@ -46,7 +46,7 @@ class << self # @api private def get_data_access_options @gdao ||= begin - input = S3Control::Client.api.operation(:get_data_access).input + input = Aws::S3Control::Client.api.operation(:get_data_access).input Set.new(input.shape.member_names) end end diff --git a/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb b/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb index 8e41e609b04..f9b11fbc887 100644 --- a/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb +++ b/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb @@ -7,7 +7,7 @@ module S3 describe AccessGrantsCredentials do let(:client) do - S3Control::Client.new(region: 'us-east-1', stub_responses: true) + Aws::S3Control::Client.new(region: 'us-east-1', stub_responses: true) end let(:in_five_minutes) { Time.now + 60 * 5 } diff --git a/tasks/rbs.rake b/tasks/rbs.rake index 7691e27cba3..b5a0ddad731 100644 --- a/tasks/rbs.rake +++ b/tasks/rbs.rake @@ -11,14 +11,15 @@ namespace :rbs do task :spytest do failures = [] # Just test s3 for most type coverage - %w[ - core - s3 - ].each do |identifier| + %w[core s3].each do |identifier| sdk_gem = "aws-sdk-#{identifier}" + ruby_opt = '-r bundler/setup -r rbs/test/setup' + if identifier == 's3' + ruby_opt += ' -I gems/aws-sdk-s3control/lib -I gems/aws-sdk-kms/lib' + end puts "Run rspec with RBS::Test on `#{sdk_gem}`" env = { - 'RUBYOPT' => '-r bundler/setup -r rbs/test/setup', + 'RUBYOPT' => ruby_opt, 'RBS_TEST_RAISE' => 'true', 'RBS_TEST_LOGLEVEL' => 'error', 'RBS_TEST_OPT' => "-I gems/aws-sdk-core/sig -I gems/#{sdk_gem}/sig", From be36879cd8e14b35befd4425427d164d3e06885c Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Mon, 29 Apr 2024 13:18:32 -0400 Subject: [PATCH 16/19] PR feedback #1 --- .../lib/aws-sdk-core/lru_cache.rb | 3 ++ gems/aws-sdk-s3/CHANGELOG.md | 2 +- .../access_grants_credentials_provider.rb | 7 +++-- .../express_credentials_provider.rb | 20 +++++++++--- .../spec/express_credentials_provider_spec.rb | 31 +++++++++++++++++-- 5 files changed, 53 insertions(+), 10 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/lru_cache.rb b/gems/aws-sdk-core/lib/aws-sdk-core/lru_cache.rb index 0f4872d9000..e14e903df2c 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/lru_cache.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/lru_cache.rb @@ -4,6 +4,9 @@ module Aws # @api private # A simple thread safe LRU cache class LRUCache + # @param [Hash] options + # @option options [Integer] :max_entries (100) Maximum number of entries + # @option options [Integer] :expiration (nil) Expiration time in seconds def initialize(options = {}) @max_entries = options[:max_entries] || 100 @expiration = options[:expiration] diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index db5ca9fa66e..289c5e0612b 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,7 +1,7 @@ Unreleased Changes ------------------ -* Feature - Support S3 Access Grants authentication. Access Grants can be enabled with the `s3_access_grants` option, and custom options can be passed into the `access_grants_credentials_provider` option. This feature is enabled if `aws-sdk-s3control` is installed. +* Feature - Support S3 Access Grants authentication. Access Grants can be enabled with the `access_grants` option, and custom options can be passed into the `access_grants_credentials_provider` option. This feature requires `aws-sdk-s3control` to be installed. 1.148.0 (2024-04-25) ------------------ diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb index 911ee6fe4fe..ab95a8c3609 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb @@ -3,9 +3,12 @@ module Aws module S3 # @api private - ACCESS_GRANTS_CREDENTIALS_CACHE = LRUCache.new + ACCESS_GRANTS_CREDENTIALS_CACHE = LRUCache.new(max_entries: 100) # @api private - ACCESS_GRANTS_ACCOUNT_ID_CACHE = LRUCache.new(expiration: 60 * 10) + ACCESS_GRANTS_ACCOUNT_ID_CACHE = LRUCache.new( + max_entries: 100, + expiration: 60 * 10 + ) # Returns Credentials class for S3 Access Grants. Accepts GetDataAccess # params and other configuration as options. See diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb index d822d058ae2..0bf005739cd 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb @@ -3,7 +3,7 @@ module Aws module S3 # @api private - EXPRESS_CREDENTIALS_CACHE = LRUCache.new + EXPRESS_CREDENTIALS_CACHE = LRUCache.new(max_entries: 100) # Returns Credentials class for S3 Express. Accepts CreateSession # params as options. See {Client#create_session} for details. @@ -18,21 +18,33 @@ class ExpressCredentialsProvider # credentials are refreshed. def initialize(options = {}) @client = options.delete(:client) - @caching = options.delete(:caching) || true + @caching = options.delete(:caching) != false @options = options @cache = EXPRESS_CREDENTIALS_CACHE end def express_credentials_for(bucket) - @cache[bucket] || new_credentials_for(bucket) + if @caching + cached_credentials_for(bucket) + else + new_credentials_for(bucket) + end end attr_accessor :client private + def cached_credentials_for(bucket) + if @cache.key?(bucket) + @cache[bucket] + else + @cache[bucket] = new_credentials_for(bucket) + end + end + def new_credentials_for(bucket) - @cache[bucket] = ExpressCredentials.new( + ExpressCredentials.new( bucket: bucket, client: @client, **@options diff --git a/gems/aws-sdk-s3/spec/express_credentials_provider_spec.rb b/gems/aws-sdk-s3/spec/express_credentials_provider_spec.rb index 138dd1fe71c..094e3249271 100644 --- a/gems/aws-sdk-s3/spec/express_credentials_provider_spec.rb +++ b/gems/aws-sdk-s3/spec/express_credentials_provider_spec.rb @@ -16,23 +16,48 @@ module S3 end describe '#express_credentials_for' do - before(:each) do + after do EXPRESS_CREDENTIALS_CACHE.clear + end + + it 'returns a new set of credentials for the bucket' do expect(ExpressCredentials).to receive(:new) .with(bucket: 'bucket', client: client, session_mode: 'ReadWrite') .and_return(express_credentials) - end - it 'returns a new set of credentials for the bucket' do expect(subject.express_credentials_for('bucket')) .to eq(express_credentials) end it 'returns the same set of credentials for the bucket' do + expect(ExpressCredentials).to receive(:new) + .with(bucket: 'bucket', client: client, session_mode: 'ReadWrite') + .and_return(express_credentials) + credentials = subject.express_credentials_for('bucket') expect(subject.express_credentials_for('bucket')) .to be(credentials) end + + context 'caching disabled' do + subject do + ExpressCredentialsProvider.new( + client: client, + session_mode: 'ReadWrite', + caching: false + ) + end + + it 'returns a different set of credentials for the bucket' do + expect(ExpressCredentials).to receive(:new) + .with(bucket: 'bucket', client: client, session_mode: 'ReadWrite') + .and_return(express_credentials).twice + + credentials = subject.express_credentials_for('bucket') + expect(subject.express_credentials_for('bucket')) + .to be(credentials) + end + end end describe '#client' do From 84373585c6d9ac28abf09e3d373675d27d52471f Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Mon, 29 Apr 2024 14:54:23 -0400 Subject: [PATCH 17/19] PR feedback #2 --- .../endpoint_provider_spec_class.mustache | 2 +- .../access_grants_credentials_provider.rb | 19 ++++++++++++------- .../express_credentials_provider.rb | 8 ++++++-- ...access_grants_credentials_provider_spec.rb | 5 +++-- .../spec/access_grants_credentials_spec.rb | 1 + .../aws-sdk-s3/spec/endpoint_provider_spec.rb | 12 ++++++------ .../spec/express_credentials_provider_spec.rb | 2 +- .../spec/plugins/express_session_auth_spec.rb | 5 ++++- 8 files changed, 34 insertions(+), 20 deletions(-) diff --git a/build_tools/aws-sdk-code-generator/templates/spec/endpoint_provider_spec_class.mustache b/build_tools/aws-sdk-code-generator/templates/spec/endpoint_provider_spec_class.mustache index 113dd74f293..1a3e9536d2c 100644 --- a/build_tools/aws-sdk-code-generator/templates/spec/endpoint_provider_spec_class.mustache +++ b/build_tools/aws-sdk-code-generator/templates/spec/endpoint_provider_spec_class.mustache @@ -58,7 +58,7 @@ module {{module_name}} expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - EXPRESS_CREDENTIALS_CACHE.clear + Aws::S3.express_credentials_cache.clear {{/s3_express_auth?}} expect_auth({{{expected_auth}}}) {{/expect_auth?}} diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb index ab95a8c3609..978e98de9b4 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb @@ -3,12 +3,17 @@ module Aws module S3 # @api private - ACCESS_GRANTS_CREDENTIALS_CACHE = LRUCache.new(max_entries: 100) + def self.access_grants_credentials_cache + @access_grants_credentials_cache ||= LRUCache.new(max_entries: 100) + end + # @api private - ACCESS_GRANTS_ACCOUNT_ID_CACHE = LRUCache.new( - max_entries: 100, - expiration: 60 * 10 - ) + def self.access_grants_account_id_cache + @access_grants_account_id_cache ||= LRUCache.new( + max_entries: 100, + expiration: 60 * 10 + ) + end # Returns Credentials class for S3 Access Grants. Accepts GetDataAccess # params and other configuration as options. See @@ -44,8 +49,8 @@ def initialize(options = {}) @bucket_region_cache = BUCKET_REGIONS # shared cache with s3_signer return unless @caching - @credentials_cache = ACCESS_GRANTS_CREDENTIALS_CACHE - @account_id_cache = ACCESS_GRANTS_ACCOUNT_ID_CACHE + @credentials_cache = Aws::S3.access_grants_credentials_cache + @account_id_cache = Aws::S3.access_grants_account_id_cache end def access_grants_credentials_for(options = {}) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb index 0bf005739cd..c24812c9f70 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/express_credentials_provider.rb @@ -3,7 +3,9 @@ module Aws module S3 # @api private - EXPRESS_CREDENTIALS_CACHE = LRUCache.new(max_entries: 100) + def self.express_credentials_cache + @express_credentials_cache ||= LRUCache.new(max_entries: 100) + end # Returns Credentials class for S3 Express. Accepts CreateSession # params as options. See {Client#create_session} for details. @@ -20,7 +22,9 @@ def initialize(options = {}) @client = options.delete(:client) @caching = options.delete(:caching) != false @options = options - @cache = EXPRESS_CREDENTIALS_CACHE + return unless @caching + + @cache = Aws::S3.express_credentials_cache end def express_credentials_for(bucket) diff --git a/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb b/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb index 69aa81cf184..a81f32f4be3 100644 --- a/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb +++ b/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +$:.unshift(File.expand_path('../../../aws-sdk-s3control/lib', __FILE__)) require_relative 'spec_helper' module Aws @@ -43,8 +44,8 @@ module S3 end after do - ACCESS_GRANTS_CREDENTIALS_CACHE.clear - ACCESS_GRANTS_ACCOUNT_ID_CACHE.clear + Aws::S3.access_grants_credentials_cache.clear + Aws::S3.access_grants_account_id_cache.clear BUCKET_REGIONS.clear end diff --git a/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb b/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb index f9b11fbc887..bf986aad8d7 100644 --- a/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb +++ b/gems/aws-sdk-s3/spec/access_grants_credentials_spec.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +$:.unshift(File.expand_path('../../../aws-sdk-s3control/lib', __FILE__)) require_relative 'spec_helper' module Aws diff --git a/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb b/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb index c835c552581..bf99a5981f5 100644 --- a/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb +++ b/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb @@ -6528,7 +6528,7 @@ module Aws::S3 expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - EXPRESS_CREDENTIALS_CACHE.clear + Aws::S3.express_credentials_cache.clear expect_auth({"name"=>"sigv4-s3express", "signingName"=>"s3express", "signingRegion"=>"us-east-1", "disableDoubleEncoding"=>true}) resp = client.get_object( bucket: 'mybucket--use1-az1--x-s3', @@ -6568,7 +6568,7 @@ module Aws::S3 expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - EXPRESS_CREDENTIALS_CACHE.clear + Aws::S3.express_credentials_cache.clear expect_auth({"name"=>"sigv4-s3express", "signingName"=>"s3express", "signingRegion"=>"us-east-1", "disableDoubleEncoding"=>true}) resp = client.get_object( bucket: 'mybucket--use1-az1--x-s3', @@ -6607,7 +6607,7 @@ module Aws::S3 expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - EXPRESS_CREDENTIALS_CACHE.clear + Aws::S3.express_credentials_cache.clear expect_auth({"name"=>"sigv4-s3express", "signingName"=>"s3express", "signingRegion"=>"ap-northeast-1", "disableDoubleEncoding"=>true}) resp = client.get_object( bucket: 'mybucket--apne1-az1--x-s3', @@ -6647,7 +6647,7 @@ module Aws::S3 expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - EXPRESS_CREDENTIALS_CACHE.clear + Aws::S3.express_credentials_cache.clear expect_auth({"name"=>"sigv4-s3express", "signingName"=>"s3express", "signingRegion"=>"ap-northeast-1", "disableDoubleEncoding"=>true}) resp = client.get_object( bucket: 'mybucket--apne1-az1--x-s3', @@ -6891,7 +6891,7 @@ module Aws::S3 expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - EXPRESS_CREDENTIALS_CACHE.clear + Aws::S3.express_credentials_cache.clear expect_auth({"name"=>"sigv4-s3express", "signingName"=>"s3express", "signingRegion"=>"us-west-2", "disableDoubleEncoding"=>true}) resp = client.get_object( bucket: 'mybucket--usw2-az1--x-s3', @@ -6945,7 +6945,7 @@ module Aws::S3 expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - EXPRESS_CREDENTIALS_CACHE.clear + Aws::S3.express_credentials_cache.clear expect_auth({"name"=>"sigv4-s3express", "signingName"=>"s3express", "signingRegion"=>"us-west-2", "disableDoubleEncoding"=>true}) resp = client.get_object( bucket: 'mybucket--usw2-az1--x-s3', diff --git a/gems/aws-sdk-s3/spec/express_credentials_provider_spec.rb b/gems/aws-sdk-s3/spec/express_credentials_provider_spec.rb index 094e3249271..ab7da0edc16 100644 --- a/gems/aws-sdk-s3/spec/express_credentials_provider_spec.rb +++ b/gems/aws-sdk-s3/spec/express_credentials_provider_spec.rb @@ -17,7 +17,7 @@ module S3 describe '#express_credentials_for' do after do - EXPRESS_CREDENTIALS_CACHE.clear + Aws::S3.express_credentials_cache.clear end it 'returns a new set of credentials for the bucket' do diff --git a/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb b/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb index 43f0738849f..60d165a3237 100644 --- a/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb +++ b/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb @@ -90,7 +90,10 @@ module S3 session_token: 's3-session', expiration: Time.now + 60 * 5 }) - EXPRESS_CREDENTIALS_CACHE.clear + end + + after do + Aws::S3.express_credentials_cache.clear end describe 's3 express auth' do From daf4d00f6b48ec751ee20bb65403d3ebbf1d2999 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Mon, 29 Apr 2024 16:17:53 -0400 Subject: [PATCH 18/19] Refactor bucket region cache --- .../lib/aws-sdk-s3/access_grants_credentials_provider.rb | 2 +- gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb | 6 +----- gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/s3_signer.rb | 9 +++++++-- .../spec/access_grants_credentials_provider_spec.rb | 2 +- gems/aws-sdk-s3/spec/client/region_detection_spec.rb | 6 +++--- gems/aws-sdk-s3/spec/client_spec.rb | 6 +++--- gems/aws-sdk-s3/spec/endpoint_provider_spec.rb | 2 +- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb index 978e98de9b4..a98e63251a9 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/access_grants_credentials_provider.rb @@ -46,7 +46,7 @@ def initialize(options = {}) @fallback = options.delete(:fallback) || false @caching = options.delete(:caching) != false @s3_control_clients = {} - @bucket_region_cache = BUCKET_REGIONS # shared cache with s3_signer + @bucket_region_cache = Aws::S3.bucket_region_cache return unless @caching @credentials_cache = Aws::S3.access_grants_credentials_cache diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb index 8ca06fb9919..07cb561f49b 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/bucket_region_cache.rb @@ -15,7 +15,7 @@ def initialize # Registers a block as a callback. This listener is called when a # new bucket/region pair is added to the cache. # - # S3::BUCKET_REGIONS.bucket_added do |bucket_name, region_name| + # Aws::S3.bucket_region_cache.bucket_added do |bucket_name, region_name| # # ... # end # @@ -81,9 +81,5 @@ def to_hash alias to_h to_hash end - - # @api private - BUCKET_REGIONS = BucketRegionCache.new - end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/s3_signer.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/s3_signer.rb index 8d18bd2abf7..d07ed5e5ea1 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/s3_signer.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/plugins/s3_signer.rb @@ -4,6 +4,11 @@ module Aws module S3 + # @api private + def self.bucket_region_cache + @bucket_region_cache ||= BucketRegionCache.new + end + module Plugins # This plugin used to have a V4 signer but it was removed in favor of # generic Sign plugin that uses endpoint auth scheme. @@ -51,7 +56,7 @@ def call(context) private def check_for_cached_region(context, bucket) - cached_region = S3::BUCKET_REGIONS[bucket] + cached_region = Aws::S3.bucket_region_cache[bucket] if cached_region && cached_region != context.config.region && !S3Signer.custom_endpoint?(context) @@ -97,7 +102,7 @@ def get_region_and_retry(context) end def update_bucket_cache(context, actual_region) - S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region + Aws::S3.bucket_region_cache[context.params[:bucket]] = actual_region end def fips_region?(resp) diff --git a/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb b/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb index a81f32f4be3..a10d0e17869 100644 --- a/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb +++ b/gems/aws-sdk-s3/spec/access_grants_credentials_provider_spec.rb @@ -46,7 +46,7 @@ module S3 after do Aws::S3.access_grants_credentials_cache.clear Aws::S3.access_grants_account_id_cache.clear - BUCKET_REGIONS.clear + Aws::S3.bucket_region_cache.clear end describe '#access_grants_credentials_for' do diff --git a/gems/aws-sdk-s3/spec/client/region_detection_spec.rb b/gems/aws-sdk-s3/spec/client/region_detection_spec.rb index d758e8f7c90..911c50b7878 100644 --- a/gems/aws-sdk-s3/spec/client/region_detection_spec.rb +++ b/gems/aws-sdk-s3/spec/client/region_detection_spec.rb @@ -33,9 +33,9 @@ module S3 'WlSrUk+8d2/rvcnEv2QXer0=' end - before(:each) do + before do allow($stderr).to receive(:write) - S3::BUCKET_REGIONS.clear + Aws::S3.bucket_region_cache.clear end context 'accessing us-west-2 bucket using classic endpoint' do @@ -93,7 +93,7 @@ module S3 end it 'does not redirect custom endpoints when the region is cached' do - S3::BUCKET_REGIONS['bucket'] = 'us-west-2' + Aws::S3.bucket_region_cache['bucket'] = 'us-west-2' stub_request(:put, 'http://bucket.localhost:9000/key') .to_return(status: [200, 'Ok']) diff --git a/gems/aws-sdk-s3/spec/client_spec.rb b/gems/aws-sdk-s3/spec/client_spec.rb index 59068229153..9f17dcf311c 100644 --- a/gems/aws-sdk-s3/spec/client_spec.rb +++ b/gems/aws-sdk-s3/spec/client_spec.rb @@ -9,7 +9,7 @@ module S3 describe Client do let(:client) { Client.new } - before(:each) do + before do Aws.config[:s3] = { region: 'us-east-1', credentials: Credentials.new('akid', 'secret'), @@ -17,9 +17,9 @@ module S3 } end - after(:each) do + after do Aws.config = {} - S3::BUCKET_REGIONS.clear + Aws::S3.bucket_region_cache.clear end it 'raises an error when region is missing' do diff --git a/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb b/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb index bf99a5981f5..b478125454e 100644 --- a/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb +++ b/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb @@ -6607,7 +6607,7 @@ module Aws::S3 expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - Aws::S3.express_credentials_cache.clear + n cachAws::S3.express_credentials_cache.clear expect_auth({"name"=>"sigv4-s3express", "signingName"=>"s3express", "signingRegion"=>"ap-northeast-1", "disableDoubleEncoding"=>true}) resp = client.get_object( bucket: 'mybucket--apne1-az1--x-s3', From 81d107e2409c3caf0341b37a564e6bb7b6751e44 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Mon, 29 Apr 2024 17:16:25 -0400 Subject: [PATCH 19/19] Undo rbs spy change as it is no longer needed --- gems/aws-sdk-s3/spec/endpoint_provider_spec.rb | 2 +- tasks/rbs.rake | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb b/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb index b478125454e..bf99a5981f5 100644 --- a/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb +++ b/gems/aws-sdk-s3/spec/endpoint_provider_spec.rb @@ -6607,7 +6607,7 @@ module Aws::S3 expiration: Time.now + 60 * 5 }) expect_auth({"name"=>"sigv4", "signingName"=>"s3express"}) - n cachAws::S3.express_credentials_cache.clear + Aws::S3.express_credentials_cache.clear expect_auth({"name"=>"sigv4-s3express", "signingName"=>"s3express", "signingRegion"=>"ap-northeast-1", "disableDoubleEncoding"=>true}) resp = client.get_object( bucket: 'mybucket--apne1-az1--x-s3', diff --git a/tasks/rbs.rake b/tasks/rbs.rake index b5a0ddad731..09faa47102a 100644 --- a/tasks/rbs.rake +++ b/tasks/rbs.rake @@ -13,13 +13,9 @@ namespace :rbs do # Just test s3 for most type coverage %w[core s3].each do |identifier| sdk_gem = "aws-sdk-#{identifier}" - ruby_opt = '-r bundler/setup -r rbs/test/setup' - if identifier == 's3' - ruby_opt += ' -I gems/aws-sdk-s3control/lib -I gems/aws-sdk-kms/lib' - end puts "Run rspec with RBS::Test on `#{sdk_gem}`" env = { - 'RUBYOPT' => ruby_opt, + 'RUBYOPT' => '-r bundler/setup -r rbs/test/setup', 'RBS_TEST_RAISE' => 'true', 'RBS_TEST_LOGLEVEL' => 'error', 'RBS_TEST_OPT' => "-I gems/aws-sdk-core/sig -I gems/#{sdk_gem}/sig",