Merge pull request #2198 from jrudolph/jr/backport-dos-fixes

Backport #2185 and #2196 to release-10.0

akka-http-core/src/main/java/akka/http/javadsl/model/HttpEntity.java

@@ -136,7 +136,7 @@ public interface HttpEntity {
     HttpEntity withoutSizeLimit();
 
     /**
-     * Returns a future of a strict entity that contains the same data as this entity
+     * Returns a CompletionStage of a strict entity that contains the same data as this entity
      * which is only completed when the complete entity has been collected. As the
      * duration of receiving the complete entity cannot be predicted, a timeout needs to
      * be specified to guard the process against running and keeping resources infinitely.
@@ -146,6 +146,17 @@ public interface HttpEntity {
      */
     CompletionStage<HttpEntity.Strict> toStrict(long timeoutMillis, Materializer materializer);
 
+    /**
+     * Returns a CompletionStage of a strict entity that contains the same data as this entity
+     * which is only completed when the complete entity has been collected. As the
+     * duration of receiving the complete entity cannot be predicted, a timeout needs to
+     * be specified to guard the process against running and keeping resources infinitely.
+     *
+     * Use getDataBytes and stream processing instead if the expected data is big or
+     * is likely to take a long time.
+     */
+    CompletionStage<HttpEntity.Strict> toStrict(long timeoutMillis, long maxBytes, Materializer materializer);
+
     /**
      * Discards the entities data bytes by running the {@code dataBytes} Source contained in this entity.
      *

akka-http-core/src/main/java/akka/http/javadsl/model/HttpMessage.java

@@ -179,7 +179,7 @@ interface MessageTransformations<Self> {
         <T> Self transformEntityDataBytes(Graph<FlowShape<ByteString, ByteString>, T> transformer);
 
         /**
-         * Returns a future of Self message with strict entity that contains the same data as this entity
+         * Returns a CompletionStage of Self message with strict entity that contains the same data as this entity
          * which is only completed when the complete entity has been collected. As the
          * duration of receiving the complete entity cannot be predicted, a timeout needs to
          * be specified to guard the process against running and keeping resources infinitely.
@@ -188,5 +188,16 @@ interface MessageTransformations<Self> {
          * is likely to take a long time.
          */
         CompletionStage<? extends Self> toStrict(long timeoutMillis, Executor ec, Materializer materializer);
+
+        /**
+         * Returns a CompletionStage of Self message with strict entity that contains the same data as this entity
+         * which is only completed when the complete entity has been collected. As the
+         * duration of receiving the complete entity cannot be predicted, a timeout needs to
+         * be specified to guard the process against running and keeping resources infinitely.
+         *
+         * Use getEntity().getDataBytes and stream processing instead if the expected data is big or
+         * is likely to take a long time.
+         */
+        CompletionStage<? extends Self> toStrict(long timeoutMillis, long maxBytes, Executor ec, Materializer materializer);
     }
 }

akka-http-core/src/main/mima-filters/10.0.13.backwards.excludes

@@ -0,0 +1,25 @@
+# Don't monitor changes to internal API
+ProblemFilters.exclude[Problem]("akka.http.impl.*")
+
+# Not meant for user extension, so new methods should be fine:
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.model.HttpEntity.toStrict")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.scaladsl.model.HttpEntity.toStrict")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.model.HttpMessage#MessageTransformations.toStrict")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.scaladsl.model.HttpMessage.toStrict")
+
+# ToStrict is private[http]
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.http.impl.util.ToStrict.this")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.settings.ParserSettings.getMaxToStrictBytes")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.settings.ParserSettings.maxToStrictBytes")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.scaladsl.settings.ParserSettings.maxToStrictBytes")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.http.impl.settings.ParserSettingsImpl.*")
+ProblemFilters.exclude[IncompatibleResultTypeProblem]("akka.http.impl.settings.ParserSettingsImpl.*")
+
+# Uri conversion additions https://github.com/akka/akka-http/pull/1950
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.model.Uri.asScala")
+
+# RoutingSettings is @DoNotInherit
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.scaladsl.settings.RoutingSettings.decodeMaxSize")
+# Impl classes
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.http.impl.settings.RoutingSettingsImpl.*")
+ProblemFilters.exclude[IncompatibleResultTypeProblem]("akka.http.impl.settings.RoutingSettingsImpl.*")

akka-http-core/src/main/resources/reference.conf

@@ -352,6 +352,10 @@ akka.http {
     # programmatically via `withSizeLimit`.)
     max-content-length = 8m
 
+    # The maximum number of bytes to allow when reading the entire entity into memory with `toStrict`
+    # (which is used by the `toStrictEntity` and `extractStrictEntity` directives)
+    max-to-strict-bytes = 8m
+
     # Sets the strictness mode for parsing request target URIs.
     # The following values are defined:
     #

akka-http-core/src/main/scala/akka/http/impl/engine/parsing/HttpMessageParser.scala

@@ -320,7 +320,7 @@ private[http] trait HttpMessageParser[Output >: MessageOutput <: ParserOutput] {
         case EntityPart(bytes)       ⇒ bytes
         case EntityStreamError(info) ⇒ throw EntityStreamException(info)
       }
-      HttpEntity.Default(contentType(cth), contentLength, HttpEntity.limitableByteSource(data))
+      HttpEntity.Default(contentType(cth), contentLength, data)
     }
 
   protected final def chunkedEntity[A <: ParserOutput](cth: Option[`Content-Type`]) =
@@ -329,7 +329,7 @@ private[http] trait HttpMessageParser[Output >: MessageOutput <: ParserOutput] {
         case EntityChunk(chunk)      ⇒ chunk
         case EntityStreamError(info) ⇒ throw EntityStreamException(info)
       }
-      HttpEntity.Chunked(contentType(cth), HttpEntity.limitableChunkSource(chunks))
+      HttpEntity.Chunked(contentType(cth), chunks)
     }
 
   protected final def addTransferEncodingWithChunkedPeeled(headers: List[HttpHeader], teh: `Transfer-Encoding`): List[HttpHeader] =

akka-http-core/src/main/scala/akka/http/impl/engine/parsing/HttpResponseParser.scala

@@ -188,7 +188,7 @@ private[http] class HttpResponseParser(protected val settings: ParserSettings, p
               emitResponseStart {
                 StreamedEntityCreator { entityParts ⇒
                   val data = entityParts.collect { case EntityPart(bytes) ⇒ bytes }
-                  HttpEntity.CloseDelimited(contentType(cth), HttpEntity.limitableByteSource(data))
+                  HttpEntity.CloseDelimited(contentType(cth), data)
                 }
               }
               setCompletionHandling(HttpMessageParser.CompletionOk)

akka-http-core/src/main/scala/akka/http/impl/settings/ParserSettingsImpl.scala

@@ -23,6 +23,7 @@ private[akka] final case class ParserSettingsImpl(
   maxHeaderValueLength:                     Int,
   maxHeaderCount:                           Int,
   maxContentLength:                         Long,
+  maxToStrictBytes:                         Long,
   maxChunkExtLength:                        Int,
   maxChunkSize:                             Int,
   uriParsingMode:                           Uri.ParsingMode,
@@ -74,6 +75,7 @@ object ParserSettingsImpl extends SettingsCompanion[ParserSettingsImpl]("akka.ht
       c getIntBytes "max-header-value-length",
       c getIntBytes "max-header-count",
       c getPossiblyInfiniteBytes "max-content-length",
+      c getPossiblyInfiniteBytes "max-to-strict-bytes",
       c getIntBytes "max-chunk-ext-length",
       c getIntBytes "max-chunk-size",
       Uri.ParsingMode(c getString "uri-parsing-mode"),

akka-http-core/src/main/scala/akka/http/impl/settings/RoutingSettingsImpl.scala

@@ -17,6 +17,7 @@ private[http] final case class RoutingSettingsImpl(
   rangeCountLimit:          Int,
   rangeCoalescingThreshold: Long,
   decodeMaxBytesPerChunk:   Int,
+  decodeMaxSize:            Long,
   fileIODispatcher:         String) extends akka.http.scaladsl.settings.RoutingSettings {
 
   override def productPrefix = "RoutingSettings"
@@ -30,5 +31,6 @@ object RoutingSettingsImpl extends SettingsCompanion[RoutingSettingsImpl]("akka.
     c getInt "range-count-limit",
     c getBytes "range-coalescing-threshold",
     c getIntBytes "decode-max-bytes-per-chunk",
+    c getPossiblyInfiniteBytes "decode-max-size",
     c getString "file-io-dispatcher")
 }

akka-http-core/src/main/scala/akka/http/impl/util/package.scala

@@ -92,11 +92,12 @@ package object util {
 
 package util {
 
-  import akka.http.scaladsl.model.{ ContentType, HttpEntity }
-  import akka.stream.{ Attributes, Outlet, Inlet, FlowShape }
+  import akka.http.scaladsl.model.{ ContentType, EntityStreamException, ErrorInfo, HttpEntity }
+  import akka.stream.{ Attributes, FlowShape, Inlet, Outlet }
+
   import scala.concurrent.duration.FiniteDuration
 
-  private[http] class ToStrict(timeout: FiniteDuration, contentType: ContentType)
+  private[http] class ToStrict(timeout: FiniteDuration, maxBytes: Option[Long], contentType: ContentType)
     extends GraphStage[FlowShape[ByteString, HttpEntity.Strict]] {
 
     val byteStringIn = Inlet[ByteString]("ToStrict.byteStringIn")
@@ -124,7 +125,12 @@ package util {
       setHandler(byteStringIn, new InHandler {
         override def onPush(): Unit = {
           bytes ++= grab(byteStringIn)
-          pull(byteStringIn)
+          maxBytes match {
+            case Some(max) if bytes.length > max ⇒
+              failStage(new EntityStreamException(new ErrorInfo("Request too large", s"Request was longer than the maximum of $max")))
+            case _ ⇒
+              pull(byteStringIn)
+          }
         }
         override def onUpstreamFinish(): Unit = {
           if (isAvailable(httpEntityOut)) {

akka-http-core/src/main/scala/akka/http/javadsl/settings/ParserSettings.scala

@@ -31,6 +31,7 @@ abstract class ParserSettings private[akka] () extends BodyPartParser.Settings {
   def getMaxHeaderValueLength: Int
   def getMaxHeaderCount: Int
   def getMaxContentLength: Long
+  def getMaxToStrictBytes: Long
   def getMaxChunkExtLength: Int
   def getMaxChunkSize: Int
   def getUriParsingMode: Uri.ParsingMode
@@ -55,6 +56,7 @@ abstract class ParserSettings private[akka] () extends BodyPartParser.Settings {
   def withMaxHeaderValueLength(newValue: Int): ParserSettings = self.copy(maxHeaderValueLength = newValue)
   def withMaxHeaderCount(newValue: Int): ParserSettings = self.copy(maxHeaderCount = newValue)
   def withMaxContentLength(newValue: Long): ParserSettings = self.copy(maxContentLength = newValue)
+  def withMaxToStrictBytes(newValue: Long): ParserSettings = self.copy(maxToStrictBytes = newValue)
   def withMaxChunkExtLength(newValue: Int): ParserSettings = self.copy(maxChunkExtLength = newValue)
   def withMaxChunkSize(newValue: Int): ParserSettings = self.copy(maxChunkSize = newValue)
   def withUriParsingMode(newValue: Uri.ParsingMode): ParserSettings = self.copy(uriParsingMode = newValue.asScala)

akka-http-core/src/main/scala/akka/http/javadsl/settings/RoutingSettings.scala

@@ -28,6 +28,7 @@ abstract class RoutingSettings private[akka] () { self: RoutingSettingsImpl ⇒
   def withRangeCountLimit(rangeCountLimit: Int): RoutingSettings = self.copy(rangeCountLimit = rangeCountLimit)
   def withRangeCoalescingThreshold(rangeCoalescingThreshold: Long): RoutingSettings = self.copy(rangeCoalescingThreshold = rangeCoalescingThreshold)
   def withDecodeMaxBytesPerChunk(decodeMaxBytesPerChunk: Int): RoutingSettings = self.copy(decodeMaxBytesPerChunk = decodeMaxBytesPerChunk)
+  def withDecodeMaxSize(decodeMaxSize: Long): RoutingSettings = self.copy(decodeMaxSize = decodeMaxSize)
   def withFileIODispatcher(fileIODispatcher: String): RoutingSettings = self.copy(fileIODispatcher = fileIODispatcher)
 }