Merge pull request #1577 from jrudolph/jr/w/1550-disable-header-parsi…

…ng-for-most-headers

+htc #1550 allow disabling of parsing to modeled headers

akka-http-core/src/main/mima-filters/10.0.10.backwards.excludes

@@ -7,3 +7,6 @@ ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.scaladsl.setting
 ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.settings.ServerSettings.getDefaultHttpPort")
 ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.settings.ServerSettings.getDefaultHttpsPort")
 
+# New settings in `@DoNotInherit` classes
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.settings.ParserSettings.getModeledHeaderParsing")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.scaladsl.settings.ParserSettings.modeledHeaderParsing")

akka-http-core/src/main/resources/reference.conf

@@ -376,6 +376,15 @@ akka.http {
     # `error-logging-verbosity`.
     illegal-header-warnings = on
 
+    # Parse headers into typed model classes in the Akka Http core layer.
+    #
+    # If set to `off`, only essential headers will be parsed into their model classes. All other ones will be provided
+    # as instances of `RawHeader`. Currently, `Connection`, `Host`, and `Expect` headers will still be provided in their
+    # typed model. The full list of headers still provided as modeled instances can be found in the source code of
+    # `akka.http.impl.engine.parsing.HttpHeaderParser.alwaysParsedHeaders`. Note that (regardless of this setting)
+    # some headers like `Content-Type` are treated specially and will never be provided in the list of headers.
+    modeled-header-parsing = on
+
     # Configures the verbosity with which message (request or response) parsing
     # errors are written to the application log.
     #

akka-http-core/src/main/scala/akka/http/impl/engine/parsing/HttpHeaderParser.scala

@@ -428,9 +428,10 @@ private[http] object HttpHeaderParser {
     def illegalHeaderWarnings: Boolean
     def illegalResponseHeaderValueProcessingMode: IllegalResponseHeaderValueProcessingMode
     def errorLoggingVerbosity: ErrorLoggingVerbosity
+    def modeledHeaderParsing: Boolean
   }
 
-  private def predefinedHeaders = Seq(
+  private val predefinedHeaders = Seq(
     "Accept: *",
     "Accept: */*",
     "Connection: Keep-Alive",
@@ -441,6 +442,15 @@ private[http] object HttpHeaderParser {
     "Cache-Control: no-cache",
     "Expect: 100-continue")
 
+  private val alwaysParsedHeaders = Set[String](
+    "connection",
+    "content-length",
+    "content-type",
+    "expect",
+    "host",
+    "transfer-encoding"
+  )
+
   def apply(settings: HttpHeaderParser.Settings, log: LoggingAdapter) =
     prime(unprimed(settings, log, defaultIllegalHeaderHandler(settings, log)))
 
@@ -454,10 +464,17 @@ private[http] object HttpHeaderParser {
     new HttpHeaderParser(settings, log, warnOnIllegalHeader)
 
   def prime(parser: HttpHeaderParser): HttpHeaderParser = {
+    val headerParserFilter: String ⇒ Boolean =
+      if (parser.settings.modeledHeaderParsing) _ ⇒ true // parse all
+      else alwaysParsedHeaders // only parse essential subset of headers
+
     val valueParsers: Seq[HeaderValueParser] =
-      HeaderParser.ruleNames.map { name ⇒
-        new ModeledHeaderValueParser(name, parser.settings.maxHeaderValueLength, parser.settings.headerValueCacheLimit(name), parser.log, parser.settings)
-      }(collection.breakOut)
+      HeaderParser.ruleNames
+        .filter(headerParserFilter)
+        .map { name ⇒
+          new ModeledHeaderValueParser(name, parser.settings.maxHeaderValueLength, parser.settings.headerValueCacheLimit(name), parser.log, parser.settings)
+        }(collection.breakOut)
+
     def insertInGoodOrder(items: Seq[Any])(startIx: Int = 0, endIx: Int = items.size): Unit =
       if (endIx - startIx > 0) {
         val pivot = (startIx + endIx) / 2
@@ -472,11 +489,13 @@ private[http] object HttpHeaderParser {
         insertInGoodOrder(items)(startIx, pivot)
         insertInGoodOrder(items)(pivot + 1, endIx)
       }
+
     insertInGoodOrder(valueParsers.sortBy(_.headerName))()
     insertInGoodOrder(specializedHeaderValueParsers)()
     insertInGoodOrder(predefinedHeaders.sorted)()
     parser.insert(ByteString("\r\n"), EmptyHeader)()
     parser.insert(ByteString("\n"), EmptyHeader)()
+
     parser
   }
 

akka-http-core/src/main/scala/akka/http/impl/settings/ParserSettingsImpl.scala

@@ -32,6 +32,7 @@ private[akka] final case class ParserSettingsImpl(
   illegalResponseHeaderValueProcessingMode: IllegalResponseHeaderValueProcessingMode,
   headerValueCacheLimits:                   Map[String, Int],
   includeTlsSessionInfoHeader:              Boolean,
+  modeledHeaderParsing:                     Boolean,
   customMethods:                            String ⇒ Option[HttpMethod],
   customStatusCodes:                        Int ⇒ Option[StatusCode],
   customMediaTypes:                         MediaTypes.FindCustom)
@@ -82,6 +83,7 @@ object ParserSettingsImpl extends SettingsCompanion[ParserSettingsImpl]("akka.ht
       IllegalResponseHeaderValueProcessingMode(c getString "illegal-response-header-value-processing-mode"),
       cacheConfig.entrySet.asScala.map(kvp ⇒ kvp.getKey → cacheConfig.getInt(kvp.getKey))(collection.breakOut),
       c getBoolean "tls-session-info-header",
+      c getBoolean "modeled-header-parsing",
       noCustomMethods,
       noCustomStatusCodes,
       noCustomMediaTypes)

akka-http-core/src/main/scala/akka/http/javadsl/settings/ParserSettings.scala

@@ -43,6 +43,7 @@ abstract class ParserSettings private[akka] () extends BodyPartParser.Settings {
   def getCustomMethods: java.util.function.Function[String, Optional[HttpMethod]]
   def getCustomStatusCodes: java.util.function.Function[Int, Optional[StatusCode]]
   def getCustomMediaTypes: akka.japi.function.Function2[String, String, Optional[MediaType]]
+  def getModeledHeaderParsing: Boolean
 
   // ---
 
@@ -61,6 +62,7 @@ abstract class ParserSettings private[akka] () extends BodyPartParser.Settings {
   def withErrorLoggingVerbosity(newValue: ParserSettings.ErrorLoggingVerbosity): ParserSettings = self.copy(errorLoggingVerbosity = newValue.asScala)
   def withHeaderValueCacheLimits(newValue: ju.Map[String, Int]): ParserSettings = self.copy(headerValueCacheLimits = newValue.asScala.toMap)
   def withIncludeTlsSessionInfoHeader(newValue: Boolean): ParserSettings = self.copy(includeTlsSessionInfoHeader = newValue)
+  def withModeledHeaderParsing(newValue: Boolean): ParserSettings = self.copy(modeledHeaderParsing = newValue)
 
   // special ---
 

akka-http-core/src/main/scala/akka/http/scaladsl/settings/ParserSettings.scala

@@ -42,6 +42,7 @@ abstract class ParserSettings private[akka] () extends akka.http.javadsl.setting
   def customMethods: String ⇒ Option[HttpMethod]
   def customStatusCodes: Int ⇒ Option[StatusCode]
   def customMediaTypes: MediaTypes.FindCustom
+  def modeledHeaderParsing: Boolean
 
   /* Java APIs */
   override def getCookieParsingMode: js.ParserSettings.CookieParsingMode = cookieParsingMode
@@ -71,8 +72,7 @@ abstract class ParserSettings private[akka] () extends akka.http.javadsl.setting
     override def apply(mainType: String, subType: String): Optional[model.MediaType] =
       OptionConverters.toJava(customMediaTypes(mainType, subType))
   }
-
-  // ---
+  def getModeledHeaderParsing: Boolean = modeledHeaderParsing
 
   // override for more specific return type
   override def withMaxUriLength(newValue: Int): ParserSettings = self.copy(maxUriLength = newValue)
@@ -86,6 +86,7 @@ abstract class ParserSettings private[akka] () extends akka.http.javadsl.setting
   override def withMaxChunkSize(newValue: Int): ParserSettings = self.copy(maxChunkSize = newValue)
   override def withIllegalHeaderWarnings(newValue: Boolean): ParserSettings = self.copy(illegalHeaderWarnings = newValue)
   override def withIncludeTlsSessionInfoHeader(newValue: Boolean): ParserSettings = self.copy(includeTlsSessionInfoHeader = newValue)
+  override def withModeledHeaderParsing(newValue: Boolean): ParserSettings = self.copy(modeledHeaderParsing = newValue)
 
   // overloads for idiomatic Scala use
   def withUriParsingMode(newValue: Uri.ParsingMode): ParserSettings = self.copy(uriParsingMode = newValue)

akka-http-core/src/test/scala/akka/http/impl/engine/parsing/HttpHeaderParserSpec.scala

@@ -243,6 +243,17 @@ abstract class HttpHeaderParserSpec(mode: String, newLine: String) extends WordS
       noException should be thrownBy parseLine(s"Server: something; something${newLine}x")
       parseAndCache(s"Server: something; something${newLine}x")() shouldEqual RawHeader("server", "something; something")
     }
+
+    "parse most headers to RawHeader when `modeled-header-parsing = off`" in new TestSetup(
+      parserSettings = createParserSettings(system).withModeledHeaderParsing(false)) {
+      // Connection, Host, and Expect should still be modelled
+      parseAndCache(s"Connection: close${newLine}x")(s"CONNECTION: close${newLine}x") shouldEqual Connection("close")
+      parseAndCache(s"Host: spray.io:123${newLine}x")(s"HOST: spray.io:123${newLine}x") shouldEqual Host("spray.io", 123)
+
+      // don't parse other headers
+      parseAndCache(s"User-Agent: hmpf${newLine}x")(s"USER-AGENT: hmpf${newLine}x") shouldEqual RawHeader("User-Agent", "hmpf")
+      parseAndCache(s"X-Forwarded-Host: localhost:8888${newLine}x")(s"X-FORWARDED-Host: localhost:8888${newLine}x") shouldEqual RawHeader("X-Forwarded-Host", "localhost:8888")
+    }
   }
 
   override def afterAll() = TestKit.shutdownActorSystem(system)

akka-http-core/src/test/scala/akka/http/scaladsl/ClientServerSpec.scala

@@ -628,6 +628,39 @@ class ClientServerSpec extends WordSpec with Matchers with BeforeAndAfterAll wit
       sinkProbe.expectComplete()
     }
 
+    "properly complete a simple request/response cycle when `modeled-header-parsing = off`" in Utils.assertAllStagesStopped {
+      new TestSetup {
+        override def configOverrides = "akka.http.parsing.modeled-header-parsing = off"
+
+        val (clientOut, clientIn) = openNewClientConnection()
+        val (serverIn, serverOut) = acceptConnection()
+
+        val clientOutSub = clientOut.expectSubscription()
+        clientOutSub.expectRequest()
+        clientOutSub.sendNext(HttpRequest(uri = "/abc"))
+
+        val serverInSub = serverIn.expectSubscription()
+        serverInSub.request(1)
+        serverIn.expectNext().uri shouldEqual Uri(s"http://$hostname:$port/abc")
+
+        val serverOutSub = serverOut.expectSubscription()
+        serverOutSub.expectRequest()
+        serverOutSub.sendNext(HttpResponse(entity = "yeah"))
+
+        val clientInSub = clientIn.expectSubscription()
+        clientInSub.request(1)
+        val response = clientIn.expectNext()
+        toStrict(response.entity) shouldEqual HttpEntity("yeah")
+
+        clientOutSub.sendComplete()
+        serverIn.expectComplete()
+        serverOutSub.expectCancellation()
+        clientIn.expectComplete()
+
+        binding.foreach(_.unbind())
+      }
+    }
+
     "be able to deal with eager closing of the request stream on the client side" in Utils.assertAllStagesStopped {
       new TestSetup {
         val (clientOut, clientIn) = openNewClientConnection()