Fix stack overflows generated when streaming requests (#538)

soto-project · Mar 7, 2023 · 787be99 · 787be99
1 parent cf1c872
commit 787be99
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 3 deletions.
diff --git a/Sources/SotoCore/HTTP/S3StreamReader.swift b/Sources/SotoCore/HTTP/S3StreamReader.swift
@@ -129,7 +129,9 @@ final class S3ChunkedStreamReader: StreamReader {
                 // if there are still bytes left to read then call _fillBuffer again, otherwise succeed with the
                 // contents of the working buffer
                 if self.bytesLeftToRead > 0 {
-                    _fillBuffer()
+                    eventLoop.execute {
+                        _fillBuffer()
+                    }
                 } else {
                     promise.succeed(self.workingBuffer)
                 }

diff --git a/Sources/SotoCore/HTTP/StreamWriter+write.swift b/Sources/SotoCore/HTTP/StreamWriter+write.swift
@@ -66,18 +66,26 @@ extension AsyncHTTPClient.HTTPClient.Body.StreamWriter {
                                     // should never reach here as HTTPClient throws HTTPClientError.bodyLengthMismatch
                                     promise.fail(AWSClient.ClientError.tooMuchData)
                                 } else {
-                                    _write(newAmountLeft)
+                                    _write(newAmountLeft, on: eventLoop)
                                 }
                             } else {
                                 _write(nil)
                             }
                             return promise.futureResult
                         }.cascadeFailure(to: promise)
                     } else {
-                        _write(newAmountLeft)
+                        _write(newAmountLeft, on: eventLoop)
                     }
                 }.cascadeFailure(to: promise)
         }
+
+        // execute write in eventLoop. Use to avoid stack overflows
+        func _write(_ amountLeft: Int?, on eventLoop: EventLoop) {
+            eventLoop.execute {
+                _write(amountLeft)
+            }
+        }
+
         _write(reader.contentSize)
 
         return promise.futureResult

diff --git a/Tests/SotoCoreTests/AWSClientTests.swift b/Tests/SotoCoreTests/AWSClientTests.swift
@@ -305,6 +305,20 @@ class AWSClientTests: XCTestCase {
         XCTAssertNoThrow(try self.testRequestStreaming(config: config, client: client, server: awsServer, bufferSize: 65552, blockSize: 65552))
     }
 
+    func testRequestStreamingAvoidStackOverflow() {
+        let awsServer = AWSTestServer(serviceProtocol: .json)
+        let httpClient = HTTPClient(eventLoopGroupProvider: .createNew)
+        let config = createServiceConfig(service: "s3", endpoint: awsServer.address)
+        let client = createAWSClient(credentialProvider: .static(accessKeyId: "foo", secretAccessKey: "bar"), httpClientProvider: .shared(httpClient))
+        defer {
+            XCTAssertNoThrow(try client.syncShutdown())
+            XCTAssertNoThrow(try awsServer.stop())
+            XCTAssertNoThrow(try httpClient.syncShutdown())
+        }
+
+        XCTAssertNoThrow(try self.testRequestStreaming(config: config, client: client, server: awsServer, bufferSize: 16 * 1024, blockSize: 8))
+    }
+
     func testRequestStreamingWithPayload(_ payload: AWSPayload) throws {
         struct Input: AWSEncodableShape & AWSShapeWithPayload {
             static var _payloadPath: String = "payload"