feat: export http server metrics

expose metrics for registration service on a separate server/port
to avoid making them available from the current Route

include a unit test to verify that the new
`sandbox_promhttp_*` metrics are exposed on the
`/metrics` endpoint

Signed-off-by: Xavier Coulon <xcoulon@redhat.com>

Signed-off-by: Xavier Coulon <xcoulon@redhat.com>

cmd/main.go

@@ -10,27 +10,26 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/codeready-toolchain/registration-service/pkg/metrics"
-	"github.com/codeready-toolchain/toolchain-common/pkg/cluster"
-	"github.com/prometheus/client_golang/prometheus"
-	controllerlog "sigs.k8s.io/controller-runtime/pkg/log"
-
 	toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1"
 	"github.com/codeready-toolchain/registration-service/pkg/auth"
 	"github.com/codeready-toolchain/registration-service/pkg/configuration"
 	"github.com/codeready-toolchain/registration-service/pkg/informers"
 	"github.com/codeready-toolchain/registration-service/pkg/log"
 	"github.com/codeready-toolchain/registration-service/pkg/proxy"
+	"github.com/codeready-toolchain/registration-service/pkg/proxy/metrics"
 	"github.com/codeready-toolchain/registration-service/pkg/server"
+	"github.com/codeready-toolchain/toolchain-common/pkg/cluster"
 	commonconfig "github.com/codeready-toolchain/toolchain-common/pkg/configuration"
 
 	errs "github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap/zapcore"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	controllerlog "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
@@ -108,12 +107,15 @@ func main() {
 		panic(errs.Wrap(err, "failed to init default token parser"))
 	}
 
-	// Register metrics
-	proxyMetrics := metrics.NewProxyMetrics(prometheus.NewRegistry())
-	// Start metrics server
-	metricsSrv := proxyMetrics.StartMetricsServer()
+	// ---------------------------------------------
+	// API Proxy
+	// ---------------------------------------------
 
-	// Start the proxy server
+	// API Proxy metrics server
+	proxyRegistry := prometheus.NewRegistry()
+	proxyMetrics := metrics.NewProxyMetrics(proxyRegistry)
+	proxyMetricsSrv := proxy.StartMetricsServer(proxyRegistry, proxy.ProxyMetricsPort)
+	// Proxy API server
 	p, err := proxy.NewProxy(app, proxyMetrics, cluster.GetMemberClusters)
 	if err != nil {
 		panic(errs.Wrap(err, "failed to create proxy"))
@@ -125,21 +127,25 @@ func main() {
 		informerShutdown <- struct{}{}
 	})
 
-	srv := server.New(app)
-
-	err = srv.SetupRoutes(proxy.DefaultPort)
+	// ---------------------------------------------
+	// Registration Service
+	// ---------------------------------------------
+	regsvcRegistry := prometheus.NewRegistry()
+	regsvcMetricsSrv, _ := server.StartMetricsServer(regsvcRegistry, server.RegSvcMetricsPort)
+	regsvcSrv := server.New(app)
+	err = regsvcSrv.SetupRoutes(proxy.DefaultPort, regsvcRegistry)
 	if err != nil {
 		panic(err.Error())
 	}
 
-	routesToPrint := srv.GetRegisteredRoutes()
+	routesToPrint := regsvcSrv.GetRegisteredRoutes()
 	log.Infof(nil, "Configured routes: %s", routesToPrint)
 
 	// listen concurrently to allow for graceful shutdown
 	go func() {
 		log.Infof(nil, "Service Revision %s built on %s", configuration.Commit, configuration.BuildTime)
 		log.Infof(nil, "Listening on %q...", configuration.HTTPAddress)
-		if err := srv.HTTPServer().ListenAndServe(); err != nil {
+		if err := regsvcSrv.HTTPServer().ListenAndServe(); err != nil {
 			if errors.Is(err, http.ErrServerClosed) {
 				log.Info(nil, fmt.Sprintf("%s - this is expected when server shutdown has been initiated", err.Error()))
 			} else {
@@ -158,7 +164,7 @@ func main() {
 		}
 	}()
 
-	gracefulShutdown(configuration.GracefulTimeout, srv.HTTPServer(), proxySrv, metricsSrv)
+	gracefulShutdown(configuration.GracefulTimeout, regsvcSrv.HTTPServer(), regsvcMetricsSrv, proxySrv, proxyMetricsSrv)
 }
 
 func gracefulShutdown(timeout time.Duration, hs ...*http.Server) {

deploy/registration-service.yaml

@@ -96,10 +96,12 @@ objects:
             - name: registration-service
               image: ${IMAGE}
               ports:
-                - containerPort: 8080
-                - containerPort: 8081
-                - containerPort: 8082
+                - containerPort: 8080 # registration service
+                - containerPort: 8081 # proxy 
+                - containerPort: 8082 # proxy metrics
                   name: metrics
+                - containerPort: 8083 # registration service metrics
+                  name: registration-service-metrics
               command:
                 - registration-service
               imagePullPolicy: IfNotPresent
@@ -140,24 +142,8 @@ objects:
                 requests:
                   cpu: "50m"
                   memory: "100M"
-  - kind: Service
-    apiVersion: v1
-    metadata:
-      name: registration-service
-      namespace: ${NAMESPACE}
-      labels:
-        provider: codeready-toolchain
-        run: registration-service
-    spec:
-      ports:
-        - name: "8080"
-          protocol: TCP
-          port: 80
-          targetPort: 8080
-      selector:
-        run: registration-service
-      type: ClusterIP
-      sessionAffinity: null
+
+  # route for the registration service
   - kind: Route
     apiVersion: v1
     metadata:
@@ -177,24 +163,48 @@ objects:
       tls:
         termination: edge
       wildcardPolicy: None
+
+  # service associated with the registration service route
   - kind: Service
     apiVersion: v1
     metadata:
-      name: api
+      name: registration-service
       namespace: ${NAMESPACE}
       labels:
         provider: codeready-toolchain
         run: registration-service
     spec:
       ports:
-        - name: "8081"
+        - name: "8080"
           protocol: TCP
           port: 80
-          targetPort: 8081
+          targetPort: 8080
+      selector:
+        run: registration-service
+      type: ClusterIP
+      sessionAffinity: null
+
+  # internal service for the registration service, used by Prometheus to scrape the metrics
+  - kind: Service
+    apiVersion: v1
+    metadata:
+      name: registration-service-metrics
+      namespace: ${NAMESPACE}
+      labels:
+        provider: codeready-toolchain
+        run: registration-service
+    spec:
+      ports:
+        - name: registration-service-metrics
+          protocol: TCP
+          port: 80
+          targetPort: registration-service-metrics
       selector:
         run: registration-service
       type: ClusterIP
       sessionAffinity: null
+
+  # route for the proxy
   - kind: Route
     apiVersion: v1
     metadata:
@@ -206,7 +216,6 @@ objects:
       name: api
       namespace: ${NAMESPACE}
     spec:
-      host: ''
       port:
         targetPort: "8081"
       to:
@@ -216,6 +225,28 @@ objects:
       tls:
         termination: edge
       wildcardPolicy: None
+
+  # service associated with the proxy route
+  - kind: Service
+    apiVersion: v1
+    metadata:
+      name: api
+      namespace: ${NAMESPACE}
+      labels:
+        provider: codeready-toolchain
+        run: registration-service
+    spec:
+      ports:
+        - name: "8081"
+          protocol: TCP
+          port: 80
+          targetPort: 8081
+      selector:
+        run: registration-service
+      type: ClusterIP
+      sessionAffinity: null
+
+  # internal service for the proxy, used by Prometheus to scrape the metrics
   - kind: Service
     apiVersion: v1
     metadata:

go.mod

@@ -33,6 +33,7 @@ require (
 	github.com/nyaruka/phonenumbers v1.1.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0
+	github.com/prometheus/common v0.40.0
 	github.com/spf13/pflag v1.0.5
 	go.uber.org/zap v1.21.0
 	gopkg.in/square/go-jose.v2 v2.3.0
@@ -73,7 +74,6 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/migueleliasweb/go-github-mock v0.0.18 // indirect
-	github.com/prometheus/common v0.40.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect

pkg/middleware/middleware_test.go → pkg/middleware/jwt_middleware_test.go

@@ -8,22 +8,22 @@ import (
 	"testing"
 	"time"
 
-	"github.com/codeready-toolchain/registration-service/test/fake"
-	"gopkg.in/h2non/gock.v1"
-
 	"github.com/codeready-toolchain/registration-service/pkg/configuration"
 	"github.com/codeready-toolchain/registration-service/pkg/middleware"
 	"github.com/codeready-toolchain/registration-service/pkg/proxy"
 	"github.com/codeready-toolchain/registration-service/pkg/server"
 	"github.com/codeready-toolchain/registration-service/test"
+	"github.com/codeready-toolchain/registration-service/test/fake"
 	"github.com/codeready-toolchain/toolchain-common/pkg/status"
 	authsupport "github.com/codeready-toolchain/toolchain-common/pkg/test/auth"
 	testconfig "github.com/codeready-toolchain/toolchain-common/pkg/test/config"
 
 	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	"gopkg.in/h2non/gock.v1"
 )
 
 type TestAuthMiddlewareSuite struct {
@@ -86,7 +86,7 @@ func (s *TestAuthMiddlewareSuite) TestAuthMiddlewareService() {
 	assert.Equal(s.T(), keysEndpointURL, cfg.Auth().AuthClientPublicKeysURL(), "key url not set correctly")
 
 	// Setting up the routes.
-	err = srv.SetupRoutes(proxy.DefaultPort)
+	err = srv.SetupRoutes(proxy.DefaultPort, prometheus.NewRegistry())
 	require.NoError(s.T(), err)
 
 	// Check that there are routes registered.

pkg/middleware/promhttp_middleware.go

@@ -0,0 +1,49 @@
+package middleware
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// see https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/promhttp#example-InstrumentRoundTripperDuration
+
+func InstrumentRoundTripperInFlight(gauge prometheus.Gauge) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		gauge.Inc()
+		defer func() {
+			gauge.Dec()
+		}()
+		c.Next()
+	}
+}
+
+func InstrumentRoundTripperCounter(counter *prometheus.CounterVec) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		defer func() {
+			counter.With(prometheus.Labels{
+				"code":   strconv.Itoa(c.Writer.Status()),
+				"method": c.Request.Method,
+				"path":   c.Request.URL.Path,
+			}).Inc()
+		}()
+		c.Next()
+	}
+}
+
+func InstrumentRoundTripperDuration(histVec *prometheus.HistogramVec) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		start := time.Now()
+		defer func() {
+			duration := time.Since(start)
+			histVec.With(prometheus.Labels{
+				"code":   strconv.Itoa(c.Writer.Status()),
+				"method": c.Request.Method,
+				"path":   c.Request.URL.Path,
+			}).Observe(float64(duration.Milliseconds()))
+		}()
+		c.Next()
+	}
+}