diff --git a/codegen/projections/white_label/lib/white_label/builders.rb b/codegen/projections/white_label/lib/white_label/builders.rb index 7ae949716..412e23422 100644 --- a/codegen/projections/white_label/lib/white_label/builders.rb +++ b/codegen/projections/white_label/lib/white_label/builders.rb @@ -69,6 +69,22 @@ def self.build(http_req, input:) end end + # Operation Builder for RequestCompressionOperation + class RequestCompressionOperation + def self.build(http_req, input:) + http_req.body = StringIO.new(input[:body] || '') + end + end + + # Operation Builder for RequestCompressionStreamingOperation + class RequestCompressionStreamingOperation + def self.build(http_req, input:) + http_req.body = input[:body] + http_req.headers['Transfer-Encoding'] = 'chunked' + http_req.headers['Content-Type'] = 'application/octet-stream' + end + end + # Operation Builder for StreamingOperation class StreamingOperation def self.build(http_req, input:) diff --git a/codegen/projections/white_label/lib/white_label/client.rb b/codegen/projections/white_label/lib/white_label/client.rb index 0181341dd..3a3890d01 100644 --- a/codegen/projections/white_label/lib/white_label/client.rb +++ b/codegen/projections/white_label/lib/white_label/client.rb @@ -722,6 +722,141 @@ def paginators_test_with_items(params = {}, options = {}, &block) resp end + # @param [Hash] params + # See {Types::RequestCompressionOperationInput}. + # + # @return [Types::RequestCompressionOperationOutput] + # + # @example Request syntax with placeholder values + # + # resp = client.request_compression_operation( + # body: 'body' + # ) + # + # @example Response structure + # + # resp.data #=> Types::RequestCompressionOperationOutput + # + def request_compression_operation(params = {}, options = {}, &block) + config = operation_config(options) + stack = Hearth::MiddlewareStack.new + input = Params::RequestCompressionOperationInput.build(params, context: 'params') + response_body = ::StringIO.new + stack.use(Hearth::Middleware::Initialize) + stack.use(Middleware::TestMiddleware, + test_config: config.test_config + ) + stack.use(Hearth::Middleware::Validate, + validator: Validators::RequestCompressionOperationInput, + validate_input: config.validate_input + ) + stack.use(Hearth::Middleware::Build, + builder: Builders::RequestCompressionOperation + ) + stack.use(Hearth::HTTP::Middleware::ContentLength) + stack.use(Hearth::HTTP::Middleware::RequestCompression, + streaming: false, + encodings: ['gzip'], + request_min_compression_size_bytes: options.fetch(:request_min_compression_size_bytes, config.request_min_compression_size_bytes), + disable_request_compression: options.fetch(:disable_request_compression, config.disable_request_compression) + ) + stack.use(Hearth::Middleware::Retry, + retry_strategy: config.retry_strategy, + error_inspector_class: Hearth::HTTP::ErrorInspector + ) + stack.use(Hearth::Middleware::Parse, + error_parser: Hearth::HTTP::ErrorParser.new(error_module: Errors, success_status: 200, errors: []), + data_parser: Parsers::RequestCompressionOperation + ) + stack.use(Hearth::Middleware::Send, + stub_responses: config.stub_responses, + client: options.fetch(:http_client, config.http_client), + stub_class: Stubs::RequestCompressionOperation, + stubs: @stubs, + params_class: Params::RequestCompressionOperationOutput + ) + resp = stack.run( + input: input, + context: Hearth::Context.new( + request: Hearth::HTTP::Request.new(uri: URI(options.fetch(:endpoint, config.endpoint))), + response: Hearth::HTTP::Response.new(body: response_body), + params: params, + logger: config.logger, + operation_name: :request_compression_operation, + interceptors: config.interceptors + ) + ) + raise resp.error if resp.error + resp + end + + # @param [Hash] params + # See {Types::RequestCompressionStreamingOperationInput}. + # + # @return [Types::RequestCompressionStreamingOperationOutput] + # + # @example Request syntax with placeholder values + # + # resp = client.request_compression_streaming_operation( + # body: 'body' + # ) + # + # @example Response structure + # + # resp.data #=> Types::RequestCompressionStreamingOperationOutput + # + def request_compression_streaming_operation(params = {}, options = {}, &block) + config = operation_config(options) + stack = Hearth::MiddlewareStack.new + input = Params::RequestCompressionStreamingOperationInput.build(params, context: 'params') + response_body = ::StringIO.new + stack.use(Hearth::Middleware::Initialize) + stack.use(Middleware::TestMiddleware, + test_config: config.test_config + ) + stack.use(Hearth::Middleware::Validate, + validator: Validators::RequestCompressionStreamingOperationInput, + validate_input: config.validate_input + ) + stack.use(Hearth::Middleware::Build, + builder: Builders::RequestCompressionStreamingOperation + ) + stack.use(Hearth::HTTP::Middleware::RequestCompression, + streaming: true, + encodings: ['gzip'], + request_min_compression_size_bytes: options.fetch(:request_min_compression_size_bytes, config.request_min_compression_size_bytes), + disable_request_compression: options.fetch(:disable_request_compression, config.disable_request_compression) + ) + stack.use(Hearth::Middleware::Retry, + retry_strategy: config.retry_strategy, + error_inspector_class: Hearth::HTTP::ErrorInspector + ) + stack.use(Hearth::Middleware::Parse, + error_parser: Hearth::HTTP::ErrorParser.new(error_module: Errors, success_status: 200, errors: []), + data_parser: Parsers::RequestCompressionStreamingOperation + ) + stack.use(Hearth::Middleware::Send, + stub_responses: config.stub_responses, + client: options.fetch(:http_client, config.http_client), + stub_class: Stubs::RequestCompressionStreamingOperation, + stubs: @stubs, + params_class: Params::RequestCompressionStreamingOperationOutput + ) + resp = stack.run( + input: input, + context: Hearth::Context.new( + request: Hearth::HTTP::Request.new(uri: URI(options.fetch(:endpoint, config.endpoint))), + response: Hearth::HTTP::Response.new(body: response_body), + params: params, + logger: config.logger, + operation_name: :request_compression_streaming_operation, + interceptors: config.interceptors + ) + ) + raise resp.error if resp.error + resp + end + # @param [Hash] params # See {Types::StreamingOperationInput}. # diff --git a/codegen/projections/white_label/lib/white_label/config.rb b/codegen/projections/white_label/lib/white_label/config.rb index 0dd636436..f27c3eda3 100644 --- a/codegen/projections/white_label/lib/white_label/config.rb +++ b/codegen/projections/white_label/lib/white_label/config.rb @@ -12,6 +12,9 @@ module WhiteLabel # @option args [Boolean] :disable_host_prefix (false) # When `true`, does not perform host prefix injection using @endpoint's hostPrefix property. # + # @option args [Boolean] :disable_request_compression (false) + # When set to 'true' the request body will not be compressed for supported operations. + # # @option args [String] :endpoint # Endpoint of the service # @@ -30,6 +33,10 @@ module WhiteLabel # @option args [Hearth::PluginList] :plugins (Hearth::PluginList.new) # A list of Plugins to apply to the client. Plugins are callables that take one argument: Config. Plugins may modify the provided config. # + # @option args [Integer] :request_min_compression_size_bytes (10240) + # The minimum size bytes that triggers compression for request bodies. + # The value must be non-negative integer value between 0 and 10485780 bytes inclusive. + # # @option args [Hearth::Retry::Strategy] :retry_strategy (Hearth::Retry::Standard.new) # Specifies which retry strategy class to use. Strategy classes # may have additional options, such as max_retries and backoff strategies. @@ -49,6 +56,9 @@ module WhiteLabel # @!attribute disable_host_prefix # @return [Boolean] # + # @!attribute disable_request_compression + # @return [Boolean] + # # @!attribute endpoint # @return [String] # @@ -67,6 +77,9 @@ module WhiteLabel # @!attribute plugins # @return [Hearth::PluginList] # + # @!attribute request_min_compression_size_bytes + # @return [Integer] + # # @!attribute retry_strategy # @return [Hearth::Retry::Strategy] # @@ -81,12 +94,14 @@ module WhiteLabel # Config = ::Struct.new( :disable_host_prefix, + :disable_request_compression, :endpoint, :http_client, :interceptors, :log_level, :logger, :plugins, + :request_min_compression_size_bytes, :retry_strategy, :stub_responses, :test_config, @@ -99,12 +114,14 @@ module WhiteLabel def validate_types! Hearth::Validator.validate_types!(disable_host_prefix, TrueClass, FalseClass, context: 'config[:disable_host_prefix]') + Hearth::Validator.validate_types!(disable_request_compression, TrueClass, FalseClass, context: 'config[:disable_request_compression]') Hearth::Validator.validate_types!(endpoint, String, context: 'config[:endpoint]') Hearth::Validator.validate_types!(http_client, Hearth::HTTP::Client, context: 'config[:http_client]') Hearth::Validator.validate_types!(interceptors, Hearth::InterceptorList, context: 'config[:interceptors]') Hearth::Validator.validate_types!(log_level, Symbol, context: 'config[:log_level]') Hearth::Validator.validate_types!(logger, Logger, context: 'config[:logger]') Hearth::Validator.validate_types!(plugins, Hearth::PluginList, context: 'config[:plugins]') + Hearth::Validator.validate_types!(request_min_compression_size_bytes, Integer, context: 'config[:request_min_compression_size_bytes]') Hearth::Validator.validate_types!(retry_strategy, Hearth::Retry::Strategy, context: 'config[:retry_strategy]') Hearth::Validator.validate_types!(stub_responses, TrueClass, FalseClass, context: 'config[:stub_responses]') Hearth::Validator.validate_types!(test_config, String, context: 'config[:test_config]') @@ -114,12 +131,14 @@ def validate_types! def self.defaults @defaults ||= { disable_host_prefix: [false], + disable_request_compression: [false], endpoint: [proc { |cfg| cfg[:stub_responses] ? 'http://localhost' : nil }], http_client: [proc { |cfg| Hearth::HTTP::Client.new(logger: cfg[:logger]) }], interceptors: [proc { Hearth::InterceptorList.new }], log_level: [:info], logger: [proc { |cfg| Logger.new($stdout, level: cfg[:log_level]) }], plugins: [proc { Hearth::PluginList.new }], + request_min_compression_size_bytes: [10240], retry_strategy: [proc { Hearth::Retry::Standard.new }], stub_responses: [false], test_config: ['default'], diff --git a/codegen/projections/white_label/lib/white_label/params.rb b/codegen/projections/white_label/lib/white_label/params.rb index 1e13f2b0f..0c527dbbe 100644 --- a/codegen/projections/white_label/lib/white_label/params.rb +++ b/codegen/projections/white_label/lib/white_label/params.rb @@ -272,6 +272,48 @@ def self.build(params, context: '') end end + module RequestCompressionOperationInput + def self.build(params, context: '') + Hearth::Validator.validate_types!(params, ::Hash, Types::RequestCompressionOperationInput, context: context) + type = Types::RequestCompressionOperationInput.new + Hearth::Validator.validate_unknown!(type, params, context: context) if params.is_a?(Hash) + type.body = params[:body] + type + end + end + + module RequestCompressionOperationOutput + def self.build(params, context: '') + Hearth::Validator.validate_types!(params, ::Hash, Types::RequestCompressionOperationOutput, context: context) + type = Types::RequestCompressionOperationOutput.new + Hearth::Validator.validate_unknown!(type, params, context: context) if params.is_a?(Hash) + type + end + end + + module RequestCompressionStreamingOperationInput + def self.build(params, context: '') + Hearth::Validator.validate_types!(params, ::Hash, Types::RequestCompressionStreamingOperationInput, context: context) + type = Types::RequestCompressionStreamingOperationInput.new + Hearth::Validator.validate_unknown!(type, params, context: context) if params.is_a?(Hash) + io = params[:body] || StringIO.new + unless io.respond_to?(:read) || io.respond_to?(:readpartial) + io = StringIO.new(io) + end + type.body = io + type + end + end + + module RequestCompressionStreamingOperationOutput + def self.build(params, context: '') + Hearth::Validator.validate_types!(params, ::Hash, Types::RequestCompressionStreamingOperationOutput, context: context) + type = Types::RequestCompressionStreamingOperationOutput.new + Hearth::Validator.validate_unknown!(type, params, context: context) if params.is_a?(Hash) + type + end + end + module ResultWrapper def self.build(params, context: '') Hearth::Validator.validate_types!(params, ::Hash, Types::ResultWrapper, context: context) diff --git a/codegen/projections/white_label/lib/white_label/parsers.rb b/codegen/projections/white_label/lib/white_label/parsers.rb index 491975939..09cfc1f0d 100644 --- a/codegen/projections/white_label/lib/white_label/parsers.rb +++ b/codegen/projections/white_label/lib/white_label/parsers.rb @@ -90,6 +90,22 @@ def self.parse(http_resp) end end + # Operation Parser for RequestCompressionOperation + class RequestCompressionOperation + def self.parse(http_resp) + data = Types::RequestCompressionOperationOutput.new + data + end + end + + # Operation Parser for RequestCompressionStreamingOperation + class RequestCompressionStreamingOperation + def self.parse(http_resp) + data = Types::RequestCompressionStreamingOperationOutput.new + data + end + end + class ResultWrapper end diff --git a/codegen/projections/white_label/lib/white_label/stubs.rb b/codegen/projections/white_label/lib/white_label/stubs.rb index 85861260d..84aeb2753 100644 --- a/codegen/projections/white_label/lib/white_label/stubs.rb +++ b/codegen/projections/white_label/lib/white_label/stubs.rb @@ -207,6 +207,32 @@ def self.stub(http_resp, stub:) end end + # Operation Stubber for RequestCompressionOperation + class RequestCompressionOperation + def self.default(visited=[]) + { + } + end + + def self.stub(http_resp, stub:) + data = {} + http_resp.status = 200 + end + end + + # Operation Stubber for RequestCompressionStreamingOperation + class RequestCompressionStreamingOperation + def self.default(visited=[]) + { + } + end + + def self.stub(http_resp, stub:) + data = {} + http_resp.status = 200 + end + end + # Structure Stubber for ResultWrapper class ResultWrapper def self.default(visited=[]) diff --git a/codegen/projections/white_label/lib/white_label/types.rb b/codegen/projections/white_label/lib/white_label/types.rb index 86ef294d7..30bed7c5e 100644 --- a/codegen/projections/white_label/lib/white_label/types.rb +++ b/codegen/projections/white_label/lib/white_label/types.rb @@ -718,6 +718,42 @@ def to_s include Hearth::Structure end + # @!attribute body + # + # @return [String] + # + RequestCompressionOperationInput = ::Struct.new( + :body, + keyword_init: true + ) do + include Hearth::Structure + end + + RequestCompressionOperationOutput = ::Struct.new( + nil, + keyword_init: true + ) do + include Hearth::Structure + end + + # @!attribute body + # + # @return [String] + # + RequestCompressionStreamingOperationInput = ::Struct.new( + :body, + keyword_init: true + ) do + include Hearth::Structure + end + + RequestCompressionStreamingOperationOutput = ::Struct.new( + nil, + keyword_init: true + ) do + include Hearth::Structure + end + # @!attribute member___123next_token # # @return [String] diff --git a/codegen/projections/white_label/lib/white_label/validators.rb b/codegen/projections/white_label/lib/white_label/validators.rb index 55c1f3624..b1a9ed019 100644 --- a/codegen/projections/white_label/lib/white_label/validators.rb +++ b/codegen/projections/white_label/lib/white_label/validators.rb @@ -270,6 +270,34 @@ def self.validate!(input, context:) end end + class RequestCompressionOperationInput + def self.validate!(input, context:) + Hearth::Validator.validate_types!(input, Types::RequestCompressionOperationInput, context: context) + Hearth::Validator.validate_types!(input[:body], ::String, context: "#{context}[:body]") + end + end + + class RequestCompressionOperationOutput + def self.validate!(input, context:) + Hearth::Validator.validate_types!(input, Types::RequestCompressionOperationOutput, context: context) + end + end + + class RequestCompressionStreamingOperationInput + def self.validate!(input, context:) + Hearth::Validator.validate_types!(input, Types::RequestCompressionStreamingOperationInput, context: context) + unless input[:body].respond_to?(:read) || input[:body].respond_to?(:readpartial) + raise ArgumentError, "Expected #{context} to be an IO like object, got #{input[:body].class}" + end + end + end + + class RequestCompressionStreamingOperationOutput + def self.validate!(input, context:) + Hearth::Validator.validate_types!(input, Types::RequestCompressionStreamingOperationOutput, context: context) + end + end + class ResultWrapper def self.validate!(input, context:) Hearth::Validator.validate_types!(input, Types::ResultWrapper, context: context) diff --git a/codegen/projections/white_label/sig/white_label/client.rbs b/codegen/projections/white_label/sig/white_label/client.rbs index 325954aee..9f0b1a36e 100644 --- a/codegen/projections/white_label/sig/white_label/client.rbs +++ b/codegen/projections/white_label/sig/white_label/client.rbs @@ -25,6 +25,8 @@ module WhiteLabel def mixin_test: (?::Hash[untyped, untyped] params, ?::Hash[untyped, untyped] options){ () -> untyped } -> untyped def paginators_test: (?::Hash[untyped, untyped] params, ?::Hash[untyped, untyped] options){ () -> untyped } -> untyped def paginators_test_with_items: (?::Hash[untyped, untyped] params, ?::Hash[untyped, untyped] options){ () -> untyped } -> untyped + def request_compression_operation: (?::Hash[untyped, untyped] params, ?::Hash[untyped, untyped] options){ () -> untyped } -> untyped + def request_compression_streaming_operation: (?::Hash[untyped, untyped] params, ?::Hash[untyped, untyped] options){ () -> untyped } -> untyped def streaming_operation: (?::Hash[untyped, untyped] params, ?::Hash[untyped, untyped] options){ () -> untyped } -> untyped def streaming_with_length: (?::Hash[untyped, untyped] params, ?::Hash[untyped, untyped] options){ () -> untyped } -> untyped def waiters_test: (?::Hash[untyped, untyped] params, ?::Hash[untyped, untyped] options){ () -> untyped } -> untyped diff --git a/codegen/projections/white_label/sig/white_label/types.rbs b/codegen/projections/white_label/sig/white_label/types.rbs index 2ed785e7e..b16da1d4d 100644 --- a/codegen/projections/white_label/sig/white_label/types.rbs +++ b/codegen/projections/white_label/sig/white_label/types.rbs @@ -40,6 +40,14 @@ module WhiteLabel PaginatorsTestWithItemsOutput: untyped + RequestCompressionOperationInput: untyped + + RequestCompressionOperationOutput: untyped + + RequestCompressionStreamingOperationInput: untyped + + RequestCompressionStreamingOperationOutput: untyped + ResultWrapper: untyped ServerError: untyped diff --git a/codegen/projections/white_label/spec/compression_spec.rb b/codegen/projections/white_label/spec/compression_spec.rb new file mode 100644 index 000000000..9bb70aa8d --- /dev/null +++ b/codegen/projections/white_label/spec/compression_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' + +module WhiteLabel + describe Config do + context 'disable_request_compression' do + it 'raises error when given an invalid input' do + expect { Config.new(disable_request_compression: 'string') } + .to raise_error( + ArgumentError, + 'Expected config[:disable_request_compression] to be in [TrueClass, FalseClass], got String.' + ) + end + end + + context 'request_min_compression_size_bytes' do + it 'raises error when given invalid integer' do + # TODO: after we implement constraints on configs + end + end + end + + describe Client do + let(:config) { Config.new(stub_responses: true) } + let(:client) { Client.new(config) } + + let(:before_send) do + Class.new do + def initialize(&block) + @block = block + end + + def read_before_transmit(context) + @block.call(context) + end + end + end + + context '#request_compression_operation' do + it 'compresses the body and sets the Content-Encoding header' do + input_body = 'a' * 10_241 + interceptor = before_send.new do |context| + uncompressed = Zlib::GzipReader.new(context.request.body) + expect(uncompressed.read).to eq(input_body) + expect(context.request.headers['Content-Encoding']).to eq('gzip') + end + client.request_compression_operation({ body: input_body }, interceptors: [interceptor]) + end + + it 'does not compress when body does not meet the minimum' do + input_body = 'Hello World' + interceptor = before_send.new do |context| + expect(context.request.body.read).to eq(input_body) + expect(context.request.headers['Content-Encoding']).to be_nil + end + client.request_compression_operation({ body: input_body }, interceptors: [interceptor]) + end + end + + context '#request_compression_streaming_operation' do + it 'compresses the streaming body and sets the Content-Encoding header' do + streaming_input = StringIO.new('Hello World') + interceptor = before_send.new do |context| + expect(context.request.headers['Content-Encoding']).to eq('gzip') + # capture the body by reading it into a new IO object + body = StringIO.new + # IO.copy_stream is the same method used by Net::Http to write our body to the socket + IO.copy_stream(context.request.body, body) + body.rewind + streaming_input.rewind + uncompressed = Zlib::GzipReader.new(body) + expect(uncompressed.read).to eq(streaming_input.read) + end + client.request_compression_streaming_operation({ body: streaming_input }, interceptors: [interceptor]) + end + end + end +end diff --git a/codegen/projections/white_label/spec/interceptor_spec.rb b/codegen/projections/white_label/spec/interceptor_spec.rb index c948f4441..ed34b6421 100644 --- a/codegen/projections/white_label/spec/interceptor_spec.rb +++ b/codegen/projections/white_label/spec/interceptor_spec.rb @@ -90,4 +90,3 @@ def read_before_execution(context); end end end end - diff --git a/codegen/projections/white_label/spec/protocol_spec.rb b/codegen/projections/white_label/spec/protocol_spec.rb index 8dd8681b9..77c399a75 100644 --- a/codegen/projections/white_label/spec/protocol_spec.rb +++ b/codegen/projections/white_label/spec/protocol_spec.rb @@ -78,6 +78,14 @@ def read_after_transmit(context) end + describe '#request_compression_operation' do + + end + + describe '#request_compression_streaming_operation' do + + end + describe '#streaming_operation' do end diff --git a/codegen/smithy-ruby-codegen-test/integration-specs/compression_spec.rb b/codegen/smithy-ruby-codegen-test/integration-specs/compression_spec.rb new file mode 100644 index 000000000..9bb70aa8d --- /dev/null +++ b/codegen/smithy-ruby-codegen-test/integration-specs/compression_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' + +module WhiteLabel + describe Config do + context 'disable_request_compression' do + it 'raises error when given an invalid input' do + expect { Config.new(disable_request_compression: 'string') } + .to raise_error( + ArgumentError, + 'Expected config[:disable_request_compression] to be in [TrueClass, FalseClass], got String.' + ) + end + end + + context 'request_min_compression_size_bytes' do + it 'raises error when given invalid integer' do + # TODO: after we implement constraints on configs + end + end + end + + describe Client do + let(:config) { Config.new(stub_responses: true) } + let(:client) { Client.new(config) } + + let(:before_send) do + Class.new do + def initialize(&block) + @block = block + end + + def read_before_transmit(context) + @block.call(context) + end + end + end + + context '#request_compression_operation' do + it 'compresses the body and sets the Content-Encoding header' do + input_body = 'a' * 10_241 + interceptor = before_send.new do |context| + uncompressed = Zlib::GzipReader.new(context.request.body) + expect(uncompressed.read).to eq(input_body) + expect(context.request.headers['Content-Encoding']).to eq('gzip') + end + client.request_compression_operation({ body: input_body }, interceptors: [interceptor]) + end + + it 'does not compress when body does not meet the minimum' do + input_body = 'Hello World' + interceptor = before_send.new do |context| + expect(context.request.body.read).to eq(input_body) + expect(context.request.headers['Content-Encoding']).to be_nil + end + client.request_compression_operation({ body: input_body }, interceptors: [interceptor]) + end + end + + context '#request_compression_streaming_operation' do + it 'compresses the streaming body and sets the Content-Encoding header' do + streaming_input = StringIO.new('Hello World') + interceptor = before_send.new do |context| + expect(context.request.headers['Content-Encoding']).to eq('gzip') + # capture the body by reading it into a new IO object + body = StringIO.new + # IO.copy_stream is the same method used by Net::Http to write our body to the socket + IO.copy_stream(context.request.body, body) + body.rewind + streaming_input.rewind + uncompressed = Zlib::GzipReader.new(body) + expect(uncompressed.read).to eq(streaming_input.read) + end + client.request_compression_streaming_operation({ body: streaming_input }, interceptors: [interceptor]) + end + end + end +end diff --git a/codegen/smithy-ruby-codegen-test/integration-specs/interceptor_spec.rb b/codegen/smithy-ruby-codegen-test/integration-specs/interceptor_spec.rb index c948f4441..ed34b6421 100644 --- a/codegen/smithy-ruby-codegen-test/integration-specs/interceptor_spec.rb +++ b/codegen/smithy-ruby-codegen-test/integration-specs/interceptor_spec.rb @@ -90,4 +90,3 @@ def read_before_execution(context); end end end end - diff --git a/codegen/smithy-ruby-codegen-test/model/component-test/compression.smithy b/codegen/smithy-ruby-codegen-test/model/component-test/compression.smithy new file mode 100644 index 000000000..ab98cad03 --- /dev/null +++ b/codegen/smithy-ruby-codegen-test/model/component-test/compression.smithy @@ -0,0 +1,31 @@ +$version: "1.0" +namespace smithy.ruby.tests + +@requestCompression( + encodings: ["gzip"] +) +@http(method: "POST", uri: "/request_compress_operation") +operation RequestCompressionOperation { + input: RequestCompressionInput +} + +@requestCompression( + encodings: ["gzip"] +) +@http(method: "POST", uri: "/request_compress_streaming_operation") +operation RequestCompressionStreamingOperation { + input: RequestCompressionStreamingInput +} + +@input +structure RequestCompressionInput { + @httpPayload + body: Blob +} + +@input +structure RequestCompressionStreamingInput { + @httpPayload + body: StreamingBlob +} + diff --git a/codegen/smithy-ruby-codegen-test/model/component-test/main.smithy b/codegen/smithy-ruby-codegen-test/model/component-test/main.smithy index e292f4da6..f0189abc3 100644 --- a/codegen/smithy-ruby-codegen-test/model/component-test/main.smithy +++ b/codegen/smithy-ruby-codegen-test/model/component-test/main.smithy @@ -18,7 +18,9 @@ service WhiteLabel { StreamingWithLength, EndpointOperation, EndpointWithHostLabelOperation, - MixinTest + MixinTest, + RequestCompressionOperation, + RequestCompressionStreamingOperation ] } diff --git a/codegen/smithy-ruby-codegen/src/main/java/software/amazon/smithy/ruby/codegen/integrations/RequestCompression.java b/codegen/smithy-ruby-codegen/src/main/java/software/amazon/smithy/ruby/codegen/integrations/RequestCompression.java new file mode 100644 index 000000000..890744502 --- /dev/null +++ b/codegen/smithy-ruby-codegen/src/main/java/software/amazon/smithy/ruby/codegen/integrations/RequestCompression.java @@ -0,0 +1,102 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.ruby.codegen.integrations; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.traits.RequestCompressionTrait; +import software.amazon.smithy.ruby.codegen.GenerationContext; +import software.amazon.smithy.ruby.codegen.RubyIntegration; +import software.amazon.smithy.ruby.codegen.config.ClientConfig; +import software.amazon.smithy.ruby.codegen.config.ConfigProviderChain; +import software.amazon.smithy.ruby.codegen.middleware.Middleware; +import software.amazon.smithy.ruby.codegen.middleware.MiddlewareBuilder; +import software.amazon.smithy.ruby.codegen.middleware.MiddlewareStackStep; +import software.amazon.smithy.ruby.codegen.util.Streaming; + +public class RequestCompression implements RubyIntegration { + @Override + public boolean includeFor(ServiceShape service, Model model) { + TopDownIndex topDownIndex = TopDownIndex.of(model); + Set containedOperations = topDownIndex.getContainedOperations(service); + return containedOperations.stream().anyMatch((o) -> o.hasTrait(RequestCompressionTrait.class)); + } + + @Override + public void modifyClientMiddleware(MiddlewareBuilder middlewareBuilder, GenerationContext context) { + ClientConfig disableRequestCompression = (new ClientConfig.Builder()) + .name("disable_request_compression") + .type("Boolean") + .defaultValue("false") + .documentation("When set to 'true' the request body will not be compressed for supported operations.") + .allowOperationOverride() + .defaults(new ConfigProviderChain.Builder() + .staticProvider("false") + .build()) + .build(); + + String minCompressionDocumentation = """ + The minimum size bytes that triggers compression for request bodies. + The value must be non-negative integer value between 0 and 10485780 bytes inclusive. + """; + + ClientConfig requestMinCompressionSizeBytes = (new ClientConfig.Builder()) + .name("request_min_compression_size_bytes") + .type("Integer") + .defaultValue("10240") + .documentation(minCompressionDocumentation) + .allowOperationOverride() + .defaults(new ConfigProviderChain.Builder() + .staticProvider("10240") + .build()) + .build(); + + Middleware compression = (new Middleware.Builder()) + .operationPredicate(((model, service, operation) -> operation.hasTrait(RequestCompressionTrait.class))) + .operationParams((ctx, operation) -> { + Map params = new HashMap<>(); + RequestCompressionTrait requestCompression = operation.expectTrait(RequestCompressionTrait.class); + Shape inputShape = ctx.model().expectShape(operation.getInputShape()); + + params.put("encodings", "[" + requestCompression + .getEncodings() + .stream() + .map((s) -> "'" + s + "'") + .collect(Collectors.joining(", ")) + "]"); + + params.put("streaming", + Streaming.isStreaming(ctx.model(), inputShape) ? "true" : "false"); + + return params; + }) + .klass("Hearth::HTTP::Middleware::RequestCompression") + .addConfig(disableRequestCompression) + .addConfig(requestMinCompressionSizeBytes) + .step(MiddlewareStackStep.AFTER_BUILD) +// commented out since Middleware Relative needs an update to handle this case +// .relative(new Middleware.Relative(Middleware.Relative.Type.BEFORE, +// "Hearth::HTTP::Middleware::ContentMD5")) + .build(); + middlewareBuilder.register(compression); + } +} diff --git a/codegen/smithy-ruby-codegen/src/main/java/software/amazon/smithy/ruby/codegen/test/protocol/fakeprotocol/generators/BuilderGenerator.java b/codegen/smithy-ruby-codegen/src/main/java/software/amazon/smithy/ruby/codegen/test/protocol/fakeprotocol/generators/BuilderGenerator.java index d42562eca..0742942ac 100644 --- a/codegen/smithy-ruby-codegen/src/main/java/software/amazon/smithy/ruby/codegen/test/protocol/fakeprotocol/generators/BuilderGenerator.java +++ b/codegen/smithy-ruby-codegen/src/main/java/software/amazon/smithy/ruby/codegen/test/protocol/fakeprotocol/generators/BuilderGenerator.java @@ -15,12 +15,15 @@ package software.amazon.smithy.ruby.codegen.test.protocol.fakeprotocol.generators; +import java.util.Optional; import software.amazon.smithy.model.shapes.ListShape; import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.StructureShape; import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.HttpPayloadTrait; import software.amazon.smithy.ruby.codegen.GenerationContext; import software.amazon.smithy.ruby.codegen.generators.BuilderGeneratorBase; import software.amazon.smithy.ruby.codegen.util.Streaming; @@ -39,12 +42,25 @@ public BuilderGenerator(GenerationContext context) { @Override protected void renderOperationBuildMethod(OperationShape operation, Shape inputShape) { writer.openBlock("def self.build(http_req, input:)"); - if (Streaming.isStreaming(model, inputShape)) { - renderStreamingBodyBuilder(inputShape); + + // checks for Payload member + Optional httpPayloadMember = inputShape.members() + .stream() + .filter((m) -> m.hasTrait(HttpPayloadTrait.class)) + .findFirst(); + if (httpPayloadMember.isPresent()) { + if (Streaming.isStreaming(model, inputShape)) { + renderStreamingBodyBuilder(inputShape); + } else { + // only works for String/Blob/Number data types + writer.write("http_req.body = StringIO.new(input[:$L] || '')", + symbolProvider.toMemberName(httpPayloadMember.get())); + } } writer.closeBlock("end"); } + @Override protected void renderStructureBuildMethod(StructureShape shape) { diff --git a/codegen/smithy-ruby-codegen/src/main/resources/META-INF/services/software.amazon.smithy.ruby.codegen.RubyIntegration b/codegen/smithy-ruby-codegen/src/main/resources/META-INF/services/software.amazon.smithy.ruby.codegen.RubyIntegration index 767416f87..a3247e18a 100755 --- a/codegen/smithy-ruby-codegen/src/main/resources/META-INF/services/software.amazon.smithy.ruby.codegen.RubyIntegration +++ b/codegen/smithy-ruby-codegen/src/main/resources/META-INF/services/software.amazon.smithy.ruby.codegen.RubyIntegration @@ -1 +1,2 @@ software.amazon.smithy.ruby.codegen.integrations.TestIntegration +software.amazon.smithy.ruby.codegen.integrations.RequestCompression \ No newline at end of file diff --git a/hearth/lib/hearth/http/middleware.rb b/hearth/lib/hearth/http/middleware.rb index 1b4596d1d..d8f189f0c 100644 --- a/hearth/lib/hearth/http/middleware.rb +++ b/hearth/lib/hearth/http/middleware.rb @@ -2,6 +2,7 @@ require_relative 'middleware/content_length' require_relative 'middleware/content_md5' +require_relative 'middleware/request_compression' module Hearth module HTTP diff --git a/hearth/lib/hearth/http/middleware/request_compression.rb b/hearth/lib/hearth/http/middleware/request_compression.rb new file mode 100644 index 000000000..57ed72804 --- /dev/null +++ b/hearth/lib/hearth/http/middleware/request_compression.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +module Hearth + module HTTP + module Middleware + # A middleware that compresses the request body and + # adds the Content-Encoding header + # @api private + class RequestCompression + SUPPORTED_ENCODINGS = %w[gzip].freeze + CHUNK_SIZE = 1 * 1024 * 1024 # one MB + + def initialize(app, + disable_request_compression:, + request_min_compression_size_bytes:, + encodings:, + streaming:) + @app = app + @disable_request_compression = disable_request_compression + @request_min_compression_size_bytes = + request_min_compression_size_bytes + @encodings = encodings + @streaming = streaming + end + + # @param input + # @param context + # @return [Output] + def call(input, context) + request = context.request + unless @disable_request_compression + selected_encoding = request_encoding_selection(@encodings) + if selected_encoding + if @streaming + process_streaming_compression(selected_encoding, request) + elsif request.body.size >= @request_min_compression_size_bytes + process_compression(selected_encoding, request) + end + end + end + @app.call(input, context) + end + + private + + def request_encoding_selection(encodings) + encodings.find { |encoding| SUPPORTED_ENCODINGS.include?(encoding) } + end + + def update_content_encoding(encoding, request) + headers = request.headers + if headers['Content-Encoding'] + headers['Content-Encoding'] += ",#{encoding}" + else + headers['Content-Encoding'] = encoding + end + end + + def process_compression(encoding, request) + case encoding + when 'gzip' + gzip_compress(request) + else + raise StandardError, 'We currently do not support ' \ + "#{encoding} encoding" + end + update_content_encoding(encoding, request) + end + + def gzip_compress(request) + compressed = StringIO.new + compressed.binmode + gzip_writer = Zlib::GzipWriter.new(compressed) + if request.body.respond_to?(:read) + update_in_chunks(gzip_writer, request.body) + else + gzip_writer.write(request.body) + end + gzip_writer.close + new_body = StringIO.new(compressed.string) + request.body = new_body + end + + def update_in_chunks(compressor, io) + loop do + chunk = io.read(CHUNK_SIZE) + break unless chunk + + compressor.write(chunk) + end + end + + def process_streaming_compression(encoding, request) + case encoding + when 'gzip' + request.body = GzipIO.new(request.body) + else + raise StandardError, 'We currently do not support ' \ + "#{encoding} encoding" + end + update_content_encoding(encoding, request) + end + + # @api private + class GzipIO + def initialize(body) + @body = body + @buffer = ChunkBuffer.new + @gzip_writer = Zlib::GzipWriter.new(@buffer) + end + + def read(length, buff = nil) + if @gzip_writer.closed? + # an empty string to signify an end as + # there will be nothing remaining to be read + StringIO.new('').read(length, buff) + return + end + + chunk = @body.read(length) + if !chunk || chunk.empty? + # closing the writer will write one last chunk + # with a trailer (to be read from the @buffer) + @gzip_writer.close + else + # flush happens first to ensure that header fields + # are being sent over since write will override + @gzip_writer.flush + @gzip_writer.write(chunk) + end + + StringIO.new(@buffer.last_chunk).read(length, buff) + end + end + + # @api private + class ChunkBuffer + def initialize + @last_chunk = nil + end + + attr_reader :last_chunk + + def write(data) + @last_chunk = data + end + end + end + end + end +end diff --git a/hearth/spec/hearth/http/middleware/request_compression_spec.rb b/hearth/spec/hearth/http/middleware/request_compression_spec.rb new file mode 100644 index 000000000..ba13721a8 --- /dev/null +++ b/hearth/spec/hearth/http/middleware/request_compression_spec.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module Hearth + module HTTP + module Middleware + describe RequestCompression do + let(:app) { double('app', call: output) } + let(:disable_request_compression) { false } + let(:request_min_compression_size_bytes) { 10_240 } # default min + let(:encodings) { ['gzip'] } # currently supported + let(:streaming) { false } + + subject do + RequestCompression.new( + app, + disable_request_compression: disable_request_compression, + request_min_compression_size_bytes: + request_min_compression_size_bytes, + encodings: encodings, + streaming: streaming + ) + end + + def expect_uncompressed_body(request, body) + expect(request.headers['Content-Encoding']).to be_nil + expect(request.body.size).to eql(body.size) + end + + describe '#call' do + let(:input) { double('input') } + let(:output) { double('output') } + let(:body) { 'a' * 10_241 } + + let(:request) do + Request.new( + http_method: 'PUT', + body: body + ) + end + + let(:response) { double('response') } + let(:context) do + Context.new( + request: request, + response: response + ) + end + + context 'disable_request_compression is true' do + let(:disable_request_compression) { true } + + it 'does not compress body' do + expect(app).to receive(:call).with(input, context) + resp = subject.call(input, context) + expect_uncompressed_body(request, body) + expect(resp).to be output + end + end + + context 'request_min_compression_size_bytes' do + context 'body size is over the minimum' do + it 'compresses the body and sets the content-encoding header' do + expect(app).to receive(:call).with(input, context) + resp = subject.call(input, context) + expect(request.headers['Content-Encoding']).to eq('gzip') + uncompressed = Zlib::GzipReader.new(request.body) + expect(uncompressed.read).to eq(body) + expect(resp).to be output + end + end + + context 'body size is less than the minimum' do + let(:body) { 'a' * 128 } + + it 'does not compress' do + expect(app).to receive(:call).with(input, context) + resp = subject.call(input, context) + expect_uncompressed_body(request, body) + expect(resp).to be output + end + end + end + + context 'no supported encodings' do + let(:encodings) { ['custom'] } + + it 'skips compression' do + expect(app).to receive(:call).with(input, context) + resp = subject.call(input, context) + expect_uncompressed_body(request, body) + expect(resp).to be output + end + end + + context 'multiple encodings' do + let(:encodings) { %w[custom gzip] } + + it 'processes the first supported encoding found' do + expect(app).to receive(:call).with(input, context) + resp = subject.call(input, context) + expect(request.headers['Content-Encoding']).to eq('gzip') + expect(resp).to be output + end + end + + context 'streaming is set true' do + let(:streaming) { true } + let(:sent_data) { StringIO.new } + + let(:app) do + proc do |_input, context| + # IO.copy_stream is the same method used by Net::Http to + # write our body to the socket + IO.copy_stream(context.request.body, sent_data) + output + end + end + + context 'a small streaming body' do + let(:body) { StringIO.new('Hello World') } + it 'compresses and preserves the original body' do + subject.call(input, context) + headers = context.request.headers + expect(headers['Content-Encoding']).to eq('gzip') + body.rewind + sent_data.rewind + uncompressed = Zlib::GzipReader.new(sent_data) + expect(uncompressed.read).to eq(body.read) + end + end + + context 'a large streaming body' do + let(:body) { StringIO.new('.' * 16_385) } + it 'compresses and preserves the original body' do + subject.call(input, context) + headers = context.request.headers + expect(headers['Content-Encoding']).to eq('gzip') + body.rewind + sent_data.rewind + uncompressed = Zlib::GzipReader.new(sent_data) + expect(uncompressed.read).to eq(body.read) + end + end + end + end + end + end + end +end