Merge pull request #461 from xmidt-org/denopink/feat/improve-event-me…

…tircs

patch: improve event delivery related metrics

http.go

@@ -125,7 +125,7 @@ func (sh *ServerHandler) ServeHTTP(response http.ResponseWriter, request *http.R
 
 func (sh *ServerHandler) recordQueueLatencyToHistogram(startTime time.Time, eventType string) {
 	endTime := sh.now()
-	sh.incomingQueueLatency.With("event", eventType).Observe(endTime.Sub(startTime).Seconds())
+	sh.incomingQueueLatency.With(eventLabel, eventType).Observe(endTime.Sub(startTime).Seconds())
 }
 
 func (sh *ServerHandler) fixWrp(msg *wrp.Message) *wrp.Message {
@@ -150,7 +150,7 @@ func (sh *ServerHandler) fixWrp(msg *wrp.Message) *wrp.Message {
 	}
 
 	if reason != "" {
-		sh.modifiedWRPCount.With("reason", reason).Add(1.0)
+		sh.modifiedWRPCount.With(reasonLabel, reason).Add(1.0)
 	}
 
 	return msg

httpClient.go

@@ -4,9 +4,13 @@
 package main
 
 import (
+	"context"
 	"errors"
+	"net"
 	"net/http"
+	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/go-kit/kit/metrics"
@@ -54,16 +58,55 @@ func (m *metricWrapper) roundTripper(next httpClient) httpClient {
 		startTime := m.now()
 		resp, err := next.Do(req)
 		endTime := m.now()
-		code := networkError
 
-		if err == nil {
+		code := genericDoReason
+		reason := noErrReason
+		if err != nil {
+			reason = getDoErrReason(err)
+			if resp != nil {
+				code = strconv.Itoa(resp.StatusCode)
+			}
+		} else {
 			code = strconv.Itoa(resp.StatusCode)
 		}
 
 		// find time difference, add to metric
-		var latency = endTime.Sub(startTime)
-		m.queryLatency.With("code", code).Observe(latency.Seconds())
+		m.queryLatency.With(urlLabel, req.URL.String(), reasonLabel, reason, codeLabel, code).Observe(endTime.Sub(startTime).Seconds())
 
 		return resp, err
 	})
 }
+
+func getDoErrReason(err error) string {
+	var d *net.DNSError
+	if err == nil {
+		return noErrReason
+	} else if errors.Is(err, context.DeadlineExceeded) {
+		return deadlineExceededReason
+	} else if errors.Is(err, context.Canceled) {
+		return contextCanceledReason
+	} else if errors.Is(err, &net.AddrError{}) {
+		return addressErrReason
+	} else if errors.Is(err, &net.ParseError{}) {
+		return parseAddrErrReason
+	} else if errors.Is(err, net.InvalidAddrError("")) {
+		return invalidAddrReason
+	} else if errors.As(err, &d) {
+		if d.IsNotFound {
+			return hostNotFoundReason
+		}
+		return dnsErrReason
+	} else if errors.Is(err, net.ErrClosed) {
+		return connClosedReason
+	} else if errors.Is(err, &net.OpError{}) {
+		return opErrReason
+	} else if errors.Is(err, net.UnknownNetworkError("")) {
+		return networkErrReason
+	} else if err, ok := err.(*url.Error); ok {
+		if strings.TrimSpace(strings.ToLower(err.Unwrap().Error())) == "eof" {
+			return connectionUnexpectedlyClosedEOFReason
+		}
+	}
+
+	return genericDoReason
+}

httpClient_test.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"strconv"
 	"testing"
 	"time"
 
@@ -16,10 +17,8 @@ import (
 )
 
 func TestRoundTripper(t *testing.T) {
-	errTest := errors.New("test error")
 	date1 := time.Date(2021, time.Month(2), 21, 1, 10, 30, 0, time.UTC)
 	date2 := time.Date(2021, time.Month(2), 21, 1, 10, 30, 45, time.UTC)
-
 	tests := []struct {
 		description      string
 		startTime        time.Time
@@ -33,32 +32,34 @@ func TestRoundTripper(t *testing.T) {
 			description:  "Success",
 			startTime:    date1,
 			endTime:      date2,
-			expectedCode: "200",
+			expectedCode: strconv.Itoa(http.StatusOK),
 			request:      exampleRequest(1),
 			expectedErr:  nil,
 			expectedResponse: &http.Response{
-				StatusCode: 200,
+				StatusCode: http.StatusOK,
 			},
 		},
 		{
 			description:  "503 Service Unavailable",
 			startTime:    date1,
 			endTime:      date2,
-			expectedCode: "503",
+			expectedCode: strconv.Itoa(http.StatusServiceUnavailable),
 			request:      exampleRequest(1),
 			expectedErr:  nil,
 			expectedResponse: &http.Response{
-				StatusCode: 503,
+				StatusCode: http.StatusServiceUnavailable,
 			},
 		},
 		{
-			description:      "Network Error",
-			startTime:        date1,
-			endTime:          date2,
-			expectedCode:     "network_err",
-			request:          exampleRequest(1),
-			expectedErr:      errTest,
-			expectedResponse: nil,
+			description:  "Network Error",
+			startTime:    date1,
+			endTime:      date2,
+			expectedCode: strconv.Itoa(http.StatusServiceUnavailable),
+			request:      exampleRequest(1),
+			expectedErr:  errors.New(genericDoReason),
+			expectedResponse: &http.Response{
+				StatusCode: http.StatusServiceUnavailable,
+			},
 		},
 	}
 
@@ -69,7 +70,7 @@ func TestRoundTripper(t *testing.T) {
 			fakeTime := mockTime(tc.startTime, tc.endTime)
 			fakeHandler := new(mockHandler)
 			fakeHist := new(mockHistogram)
-			histogramFunctionCall := []string{"code", tc.expectedCode}
+			histogramFunctionCall := []string{urlLabel, tc.request.URL.String(), reasonLabel, getDoErrReason(tc.expectedErr), codeLabel, tc.expectedCode}
 			fakeLatency := date2.Sub(date1)
 			fakeHist.On("With", histogramFunctionCall).Return().Once()
 			fakeHist.On("Observe", fakeLatency.Seconds()).Return().Once()

http_test.go

@@ -106,7 +106,7 @@ func TestServerHandler(t *testing.T) {
 
 		fakeTime := mockTime(tc.startTime, tc.endTime)
 		fakeHist := new(mockHistogram)
-		histogramFunctionCall := []string{"event", tc.expectedEventType}
+		histogramFunctionCall := []string{eventLabel, tc.expectedEventType}
 		fakeLatency := date2.Sub(date1)
 		fakeHist.On("With", histogramFunctionCall).Return().Once()
 		fakeHist.On("Observe", fakeLatency.Seconds()).Return().Once()
@@ -162,11 +162,11 @@ func TestServerHandlerFixWrp(t *testing.T) {
 	fakeIncomingContentTypeCount.On("Add", 1.0).Return()
 
 	fakeModifiedWRPCount := new(mockCounter)
-	fakeModifiedWRPCount.On("With", []string{"reason", bothEmptyReason}).Return(fakeIncomingContentTypeCount).Once()
+	fakeModifiedWRPCount.On("With", []string{reasonLabel, bothEmptyReason}).Return(fakeIncomingContentTypeCount).Once()
 	fakeModifiedWRPCount.On("Add", 1.0).Return().Once()
 
 	fakeHist := new(mockHistogram)
-	histogramFunctionCall := []string{"event", "bob"}
+	histogramFunctionCall := []string{eventLabel, "bob"}
 	fakeLatency := date2.Sub(date1)
 	fakeHist.On("With", histogramFunctionCall).Return().Once()
 	fakeHist.On("Observe", fakeLatency.Seconds()).Return().Once()
@@ -215,7 +215,7 @@ func TestServerHandlerFull(t *testing.T) {
 	fakeQueueDepth.On("Add", mock.AnythingOfType("float64")).Return().Times(4)
 
 	fakeHist := new(mockHistogram)
-	histogramFunctionCall := []string{"event", unknownEventType}
+	histogramFunctionCall := []string{eventLabel, unknownEventType}
 	fakeLatency := date2.Sub(date1)
 	fakeHist.On("With", histogramFunctionCall).Return().Once()
 	fakeHist.On("Observe", fakeLatency.Seconds()).Return().Once()
@@ -270,7 +270,7 @@ func TestServerEmptyPayload(t *testing.T) {
 	fakeQueueDepth.On("Add", mock.AnythingOfType("float64")).Return().Times(4)
 
 	fakeHist := new(mockHistogram)
-	histogramFunctionCall := []string{"event", unknownEventType}
+	histogramFunctionCall := []string{eventLabel, unknownEventType}
 	fakeLatency := date2.Sub(date1)
 	fakeHist.On("With", histogramFunctionCall).Return().Once()
 	fakeHist.On("Observe", fakeLatency.Seconds()).Return().Once()
@@ -325,7 +325,7 @@ func TestServerUnableToReadBody(t *testing.T) {
 	fakeQueueDepth.On("Add", mock.AnythingOfType("float64")).Return().Times(4)
 
 	fakeHist := new(mockHistogram)
-	histogramFunctionCall := []string{"event", unknownEventType}
+	histogramFunctionCall := []string{eventLabel, unknownEventType}
 	fakeLatency := date2.Sub(date1)
 	fakeHist.On("With", histogramFunctionCall).Return().Once()
 	fakeHist.On("Observe", fakeLatency.Seconds()).Return().Once()
@@ -380,7 +380,7 @@ func TestServerInvalidBody(t *testing.T) {
 	fakeInvalidCount.On("Add", mock.AnythingOfType("float64")).Return().Once()
 
 	fakeHist := new(mockHistogram)
-	histogramFunctionCall := []string{"event", unknownEventType}
+	histogramFunctionCall := []string{eventLabel, unknownEventType}
 	fakeLatency := date2.Sub(date1)
 	fakeHist.On("With", histogramFunctionCall).Return().Once()
 	fakeHist.On("Observe", fakeLatency.Seconds()).Return().Once()
@@ -415,7 +415,7 @@ func TestHandlerUnsupportedMediaType(t *testing.T) {
 	date1 := time.Date(2021, time.Month(2), 21, 1, 10, 30, 0, time.UTC)
 	date2 := time.Date(2021, time.Month(2), 21, 1, 10, 30, 45, time.UTC)
 
-	histogramFunctionCall := []string{"event", unknownEventType}
+	histogramFunctionCall := []string{eventLabel, unknownEventType}
 	fakeLatency := date2.Sub(date1)
 
 	assert := assert.New(t)