Http compression support (#335)

* feat: gzip compression

* fix brotli content-encoding header

* fix brotli encoding-header test

* fix h2 tests

cmd/flags/grpc_test.go

@@ -15,9 +15,10 @@
 package flags
 
 import (
+	"testing"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"testing"
 )
 
 func TestGrpc_ToGrpcRequests(t *testing.T) {

cmd/flags/http.go

@@ -34,7 +34,8 @@ var allowedHTTPMethods = map[string]interface{}{
 
 // HTTP stores flags related to HTTP requests.
 type HTTP struct {
-	Requests stringArray
+	Requests    stringArray
+	Compression string
 }
 
 func (h *HTTP) String() string {
@@ -43,16 +44,17 @@ func (h *HTTP) String() string {
 
 func (h *HTTP) initFlags() {
 	flag.Var(&h.Requests, "http-requests", `HTTP request to be sent. Request is in '<http-method>:<path>[:body]' format. E.g. post:/ping:{"key":"value"}`)
+	flag.StringVar(&h.Compression, "http-requests-compression", "", "Compression is disabled by default. Allows compression of Http body either with `gzip`, `deflate` or `brotli`. Using one of the compression algorithms also the according `Content-Encoding` header is added.")
 }
 
 func (h *HTTP) getWarmupHTTPRequests() ([]http.Request, error) {
-	return toHTTPRequests(h.Requests)
+	return toHTTPRequests(h.Requests, http.CompressionType(h.Compression))
 }
 
-func toHTTPRequests(requestsFlag []string) ([]http.Request, error) {
+func toHTTPRequests(requestsFlag []string, compression http.CompressionType) ([]http.Request, error) {
 	var requests []http.Request
 	for _, requestFlag := range requestsFlag {
-		request, err := http.ToHTTPRequest(requestFlag)
+		request, err := http.ToHTTPRequest(requestFlag, compression)
 		if err != nil {
 			return nil, err
 		}

cmd/flags/http_test.go

@@ -15,10 +15,11 @@
 package flags
 
 import (
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"mittens/internal/pkg/http"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestHttp_ToHttpRequests(t *testing.T) {
@@ -27,7 +28,7 @@ func TestHttp_ToHttpRequests(t *testing.T) {
 		"get:/ping",
 	}
 
-	requests, err := toHTTPRequests(requestFlags)
+	requests, err := toHTTPRequests(requestFlags, http.COMPRESSION_NONE)
 	require.NoError(t, err)
 
 	require.Equal(t, 2, len(requests))
@@ -41,7 +42,7 @@ func TestHttp_ToHttpRequestsInvalidFormat(t *testing.T) {
 		"get/health",
 	}
 
-	requests, err := toHTTPRequests(requestFlags)
+	requests, err := toHTTPRequests(requestFlags, http.COMPRESSION_NONE)
 
 	var expected []http.Request
 	require.Error(t, err)
@@ -55,7 +56,7 @@ func TestHttp_ToHttpRequestsInvalidMethod(t *testing.T) {
 		"invalidMethod:/health",
 	}
 
-	requests, err := toHTTPRequests(requestFlags)
+	requests, err := toHTTPRequests(requestFlags, http.COMPRESSION_NONE)
 
 	var expected []http.Request
 	require.Error(t, err)
@@ -69,7 +70,7 @@ func TestHttp_ToHttpRequestsInvalidBody(t *testing.T) {
 		"get:/test:file:test",
 	}
 
-	requests, err := toHTTPRequests(requestFlags)
+	requests, err := toHTTPRequests(requestFlags, http.COMPRESSION_NONE)
 
 	var expected []http.Request
 	require.Error(t, err)

docs/about/getting-started.md

@@ -18,6 +18,7 @@ The application receives a number of command-line flags including the requests t
 | -http-headers                                                  | strings | N/A                         | Http headers to be sent with warm up requests. To send multiple headers define this flag for each header                                                                                                                                                                                |
 | -grpc-requests                                                 | strings | N/A                         | gRPC requests to be sent. Request is in '\<service\>\<method\>\[:message\]' format. E.g. health/ping:{"key": "value"}. To send multiple requests, simply repeat this flag for each request. Use the notation `:file/xyz.json` if you want to use an external file for the request body. |
 | -http-requests                                                 | string  | N/A                         | Http request to be sent. Request is in `<http-method>:<path>[:body]` format. E.g. `post:/ping:{"key": "value"}`. To send multiple requests, simply repeat this flag for each request. Use the notation `:file/xyz.json` if you want to use an external file for the request body.       |
+| -http-requests-compression                                     | string  | N/A                         | Compression is disabled by default. Allows compression of Http body either with `gzip`, `deflate` or `brotli`. Using one of the compression algorithms also the according `Content-Encoding` header is added.                                                                           |
 | -fail-readiness                                                | bool    | false                       | If set to true readiness will fail if the target did not became ready in time                                                                                                                                                                                                           |
 | -file-probe-enabled                                            | bool    | true                        | If set to true writes files that can be used as readiness/liveness probes. a file with the name `alive` is created when Mittens starts and a file named `ready` is created when the warmup completes                                                                                    |
 | -file-probe-liveness-path                                      | string  | alive                       | File to be used for liveness probe                                                                                                                                                                                                                                                      |

go.mod

@@ -1,6 +1,7 @@
 module mittens
 
 require (
+	github.com/andybalholm/brotli v1.0.6
 	github.com/fullstorydev/grpcurl v1.8.9
 	github.com/golang/protobuf v1.5.3
 	github.com/jhump/protoreflect v1.15.4

go.sum

@@ -1,3 +1,5 @@
+github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
+github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/bufbuild/protocompile v0.7.1 h1:Kd8fb6EshOHXNNRtYAmLAwy/PotlyFoN0iMbuwGNh0M=
 github.com/bufbuild/protocompile v0.7.1/go.mod h1:+Etjg4guZoAqzVk2czwEQP12yaxLJ8DxuqCJ9qHdH94=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

internal/pkg/http/client.go

@@ -15,15 +15,13 @@
 package http
 
 import (
-	"bytes"
 	"crypto/tls"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"mittens/internal/pkg/placeholders"
 	"mittens/internal/pkg/response"
-	"mittens/internal/pkg/util"
 	"net"
 	"net/http"
 	"strings"
@@ -75,12 +73,8 @@ func NewClient(host string, insecure bool, timeoutMilliseconds int, protocol Pro
 }
 
 // SendRequest sends a request to the HTTP server and wraps useful information into a Response object.
-func (c Client) SendRequest(method, path string, headers []string, requestBody *string) response.Response {
+func (c Client) SendRequest(method, path string, headers map[string]string, body io.Reader) response.Response {
 	const respType = "http"
-	var body io.Reader
-	if requestBody != nil {
-		body = bytes.NewBufferString(*requestBody)
-	}
 
 	url := fmt.Sprintf("%s/%s", c.host, strings.TrimLeft(path, "/"))
 	req, err := http.NewRequest(method, url, body)
@@ -90,8 +84,7 @@ func (c Client) SendRequest(method, path string, headers []string, requestBody *
 		return response.Response{Duration: time.Duration(0), Err: err, Type: respType}
 	}
 
-	headersMap := util.ToHeaders(headers)
-	for k, v := range headersMap {
+	for k, v := range headers {
 		if strings.EqualFold(k, "Host") {
 			req.Host = v
 		}
@@ -101,9 +94,14 @@ func (c Client) SendRequest(method, path string, headers []string, requestBody *
 
 		req.Header.Add(k, interpolatedHeaderValue)
 	}
-
+	if err != nil {
+		defer req.Body.Close()
+	}
 	startTime := time.Now()
 	resp, err := c.httpClient.Do(req)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
 	endTime := time.Now()
 	if err != nil {
 		return response.Response{Duration: endTime.Sub(startTime), Err: err, Type: respType}

internal/pkg/http/client_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"mittens/fixture"
 	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -39,44 +40,50 @@ func TestMain(m *testing.M) {
 func TestRequestSuccessHTTP1(t *testing.T) {
 	c := NewClient(serverUrl, false, 10000, HTTP1)
 	reqBody := ""
-	resp := c.SendRequest("GET", WorkingPath, []string{}, &reqBody)
+	reader := strings.NewReader(reqBody)
+	resp := c.SendRequest("GET", WorkingPath, make(map[string]string), reader)
 	assert.Nil(t, resp.Err)
 }
 
 func TestRequestSuccessH2C(t *testing.T) {
 	c := NewClient(serverUrl, false, 10000, H2C)
 	reqBody := ""
-	resp := c.SendRequest("GET", WorkingPath, []string{}, &reqBody)
+	reader := strings.NewReader(reqBody)
+	resp := c.SendRequest("GET", WorkingPath, make(map[string]string), reader)
 	assert.Nil(t, resp.Err)
 }
 
 func TestHttpErrorHTTP1(t *testing.T) {
 	c := NewClient(serverUrl, false, 10000, HTTP1)
 	reqBody := ""
-	resp := c.SendRequest("GET", "/", []string{}, &reqBody)
+	reader := strings.NewReader(reqBody)
+	resp := c.SendRequest("GET", "/", make(map[string]string), reader)
 	assert.Nil(t, resp.Err)
 	assert.Equal(t, resp.StatusCode, 404)
 }
 
 func TestHttpErrorH2C(t *testing.T) {
 	c := NewClient(serverUrl, false, 10000, H2C)
 	reqBody := ""
-	resp := c.SendRequest("GET", "/", []string{}, &reqBody)
+	reader := strings.NewReader(reqBody)
+	resp := c.SendRequest("GET", "/", make(map[string]string), reader)
 	assert.Nil(t, resp.Err)
 	assert.Equal(t, resp.StatusCode, 404)
 }
 
 func TestConnectionErrorHTTP1(t *testing.T) {
 	c := NewClient("http://localhost:9999", false, 10000, HTTP1)
 	reqBody := ""
-	resp := c.SendRequest("GET", "/potato", []string{}, &reqBody)
+	reader := strings.NewReader(reqBody)
+	resp := c.SendRequest("GET", "/potato", make(map[string]string), reader)
 	assert.NotNil(t, resp.Err)
 }
 
 func TestConnectionErrorH2C(t *testing.T) {
 	c := NewClient("http://localhost:9999", false, 10000, H2C)
 	reqBody := ""
-	resp := c.SendRequest("GET", "/potato", []string{}, &reqBody)
+	reader := strings.NewReader(reqBody)
+	resp := c.SendRequest("GET", "/potato", make(map[string]string), reader)
 	assert.NotNil(t, resp.Err)
 }
 

internal/pkg/http/utils.go

@@ -15,18 +15,34 @@
 package http
 
 import (
+	"bytes"
+	"compress/flate"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"mittens/internal/pkg/placeholders"
 	"strings"
+
+	"github.com/andybalholm/brotli"
 )
 
 // Request represents an HTTP request.
 type Request struct {
-	Method string
-	Path   string
-	Body   *string
+	Method  string
+	Headers map[string]string
+	Path    string
+	Body    io.Reader
 }
 
+type CompressionType string
+
+const (
+	COMPRESSION_NONE    CompressionType = ""
+	COMPRESSION_GZIP    CompressionType = "gzip"
+	COMPRESSION_BROTLI  CompressionType = "brotli"
+	COMPRESSION_DEFLATE CompressionType = "deflate"
+)
+
 var allowedHTTPMethods = map[string]interface{}{
 	"GET":     nil,
 	"HEAD":    nil,
@@ -39,9 +55,8 @@ var allowedHTTPMethods = map[string]interface{}{
 	"TRACE":   nil,
 }
 
-//
 // ToHTTPRequest parses an HTTP request which is in a string format and stores it in a struct.
-func ToHTTPRequest(requestString string) (Request, error) {
+func ToHTTPRequest(requestString string, compression CompressionType) (Request, error) {
 	parts := strings.SplitN(requestString, ":", 3)
 	if len(parts) < 2 {
 		return Request{}, fmt.Errorf("invalid request flag: %s, expected format <http-method>:<path>[:body]", requestString)
@@ -72,9 +87,63 @@ func ToHTTPRequest(requestString string) (Request, error) {
 	}
 	var body = placeholders.InterpolatePlaceholders(*rawBody)
 
+	var reader io.Reader
+	switch compression {
+	case COMPRESSION_GZIP:
+		reader = compressGzip([]byte(body))
+	case COMPRESSION_BROTLI:
+		reader = compressBrotli([]byte(body))
+	case COMPRESSION_DEFLATE:
+		reader = compressFlate([]byte(body))
+	default:
+		reader = bytes.NewBufferString(body)
+	}
+
+	headers := make(map[string]string)
+	if compression != COMPRESSION_NONE {
+		encoding := ""
+		switch compression {
+		case COMPRESSION_GZIP:
+			encoding = "gzip"
+		case COMPRESSION_BROTLI:
+			encoding = "br"
+		case COMPRESSION_DEFLATE:
+			encoding = "deflate"
+		}
+		headers["Content-Encoding"] = encoding
+	}
+
 	return Request{
-		Method: method,
-		Path:   path,
-		Body:   &body,
+		Method:  method,
+		Headers: headers,
+		Path:    path,
+		Body:    reader,
 	}, nil
 }
+
+func compressGzip(data []byte) io.Reader {
+	pr, pw := io.Pipe()
+	go func() {
+		gz := gzip.NewWriter(pw)
+		_, err := gz.Write(data)
+		gz.Close()
+		pw.CloseWithError(err)
+	}()
+	return pr
+}
+
+func compressFlate(data []byte) *bytes.Buffer {
+	var b bytes.Buffer
+	w, _ := flate.NewWriter(&b, 9)
+	w.Write(data)
+	w.Close()
+	return &b
+}
+
+func compressBrotli(data []byte) *bytes.Buffer {
+	var b bytes.Buffer
+	w := brotli.NewWriterLevel(&b, brotli.BestCompression)
+	w.Write(data)
+	w.Close()
+	return &b
+}