diff --git a/cmd/main.go b/cmd/main.go index baf30732..7d906713 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,27 +10,26 @@ import ( "syscall" "time" - "github.com/codeready-toolchain/registration-service/pkg/metrics" - "github.com/codeready-toolchain/toolchain-common/pkg/cluster" - "github.com/prometheus/client_golang/prometheus" - controllerlog "sigs.k8s.io/controller-runtime/pkg/log" - toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1" "github.com/codeready-toolchain/registration-service/pkg/auth" "github.com/codeready-toolchain/registration-service/pkg/configuration" "github.com/codeready-toolchain/registration-service/pkg/informers" "github.com/codeready-toolchain/registration-service/pkg/log" + "github.com/codeready-toolchain/registration-service/pkg/metrics" "github.com/codeready-toolchain/registration-service/pkg/proxy" "github.com/codeready-toolchain/registration-service/pkg/server" + "github.com/codeready-toolchain/toolchain-common/pkg/cluster" commonconfig "github.com/codeready-toolchain/toolchain-common/pkg/configuration" errs "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap/zapcore" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" + controllerlog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) diff --git a/go.mod b/go.mod index b046d122..01c806dd 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/nyaruka/phonenumbers v1.1.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 + github.com/prometheus/common v0.40.0 github.com/spf13/pflag v1.0.5 go.uber.org/zap v1.21.0 gopkg.in/square/go-jose.v2 v2.3.0 @@ -73,7 +74,6 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/migueleliasweb/go-github-mock v0.0.18 // indirect - github.com/prometheus/common v0.40.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect diff --git a/pkg/middleware/middleware.go b/pkg/middleware/jwt_middleware.go similarity index 100% rename from pkg/middleware/middleware.go rename to pkg/middleware/jwt_middleware.go diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/jwt_middleware_test.go similarity index 100% rename from pkg/middleware/middleware_test.go rename to pkg/middleware/jwt_middleware_test.go diff --git a/pkg/middleware/promhttp_middleware.go b/pkg/middleware/promhttp_middleware.go new file mode 100644 index 00000000..7e681684 --- /dev/null +++ b/pkg/middleware/promhttp_middleware.go @@ -0,0 +1,49 @@ +package middleware + +import ( + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" +) + +// see https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/promhttp#example-InstrumentRoundTripperDuration + +func InstrumentRoundTripperInFlight(gauge prometheus.Gauge) gin.HandlerFunc { + return func(c *gin.Context) { + gauge.Inc() + defer func() { + gauge.Dec() + }() + c.Next() + } +} + +func InstrumentRoundTripperCounter(counter *prometheus.CounterVec) gin.HandlerFunc { + return func(c *gin.Context) { + defer func() { + counter.With(prometheus.Labels{ + "code": strconv.Itoa(c.Writer.Status()), + "method": c.Request.Method, + "path": c.Request.URL.Path, + }).Inc() + }() + c.Next() + } +} + +func InstrumentRoundTripperDuration(histVec *prometheus.HistogramVec) gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + defer func() { + duration := time.Since(start) + histVec.With(prometheus.Labels{ + "code": strconv.Itoa(c.Writer.Status()), + "method": c.Request.Method, + "path": c.Request.URL.Path, + }).Observe(float64(duration.Milliseconds())) + }() + c.Next() + } +} diff --git a/pkg/middleware/promhttp_middleware_test.go b/pkg/middleware/promhttp_middleware_test.go new file mode 100644 index 00000000..c3a76c29 --- /dev/null +++ b/pkg/middleware/promhttp_middleware_test.go @@ -0,0 +1,123 @@ +package middleware_test + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/codeready-toolchain/registration-service/pkg/configuration" + "github.com/codeready-toolchain/registration-service/pkg/proxy" + "github.com/codeready-toolchain/registration-service/pkg/server" + "github.com/codeready-toolchain/registration-service/test" + "github.com/codeready-toolchain/registration-service/test/fake" + authsupport "github.com/codeready-toolchain/toolchain-common/pkg/test/auth" + testconfig "github.com/codeready-toolchain/toolchain-common/pkg/test/config" + + prommodel "github.com/prometheus/client_model/go" + promcommon "github.com/prometheus/common/expfmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type PromHTTPMiddlewareSuite struct { + test.UnitTestSuite +} + +func TestPromHTTPMiddlewareSuite(t *testing.T) { + suite.Run(t, &PromHTTPMiddlewareSuite{test.UnitTestSuite{}}) +} + +func (s *PromHTTPMiddlewareSuite) TestPromHTTPMiddleware() { + // given + tokengenerator := authsupport.NewTokenManager() + srv := server.New(fake.NewMockableApplication(nil)) + keysEndpointURL := tokengenerator.NewKeyServer().URL + s.SetConfig(testconfig.RegistrationService(). + Environment(configuration.UnitTestsEnvironment). + Auth().AuthClientPublicKeysURL(keysEndpointURL)) + + cfg := configuration.GetRegistrationServiceConfig() + assert.Equal(s.T(), keysEndpointURL, cfg.Auth().AuthClientPublicKeysURL(), "key url not set correctly") + err := srv.SetupRoutes(proxy.DefaultPort) + require.NoError(s.T(), err) + + s.Run("call endpoint", func() { + // making a call on an HTTP endpoint + resp := httptest.NewRecorder() + req, err := http.NewRequest(http.MethodGet, "/api/v1/segment-write-key", nil) + require.NoError(s.T(), err) + + // when + srv.Engine().ServeHTTP(resp, req) + + // then + assert.Equal(s.T(), http.StatusOK, resp.Code, "request returned wrong status code") + + s.Run("check metrics", func() { + // also, we should have some HTTP metrics + + resp = httptest.NewRecorder() + req, err = http.NewRequest(http.MethodGet, "/metrics", nil) + require.NoError(s.T(), err) + + // when + srv.Engine().ServeHTTP(resp, req) + + // then + assert.Equal(s.T(), http.StatusOK, resp.Code, "request returned wrong status code") + require.NoError(s.T(), err) + + _, err := getMetric(resp.Body.Bytes(), "sandbox_promhttp_client_in_flight_requests", nil) + assert.NoError(s.T(), err) + + _, err = getMetric(resp.Body.Bytes(), "sandbox_promhttp_client_api_requests_total", map[string]string{ + "code": "200", + "method": "GET", + "path": "/api/v1/segment-write-key", + }) + assert.NoError(s.T(), err) + + _, err = getMetric(resp.Body.Bytes(), "sandbox_promhttp_request_duration_seconds", map[string]string{ + "code": "200", + "method": "GET", + "path": "/api/v1/segment-write-key", + }) + assert.NoError(s.T(), err) + }) + }) +} + +func getMetric(data []byte, name string, labels map[string]string) (*prommodel.Metric, error) { + p := &promcommon.TextParser{} + metrics, err := p.TextToMetricFamilies(bytes.NewReader(data)) + if err != nil { + return nil, err + } + + metric, found := metrics[name] + if !found { + return nil, fmt.Errorf("unable to find metric '%s'", name) + } + for _, m := range metric.Metric { + if matchesLabels(m.Label, labels) { + return m, nil + } + } + + return nil, fmt.Errorf("unable to find metric '%s' with labels '%v'", name, labels) +} + +func matchesLabels(x []*prommodel.LabelPair, y map[string]string) bool { + if len(x) != len(y) { + return false + } + for _, l := range x { + if v, found := y[*l.Name]; !found || *l.Value != v { + return false + } + } + return true +} diff --git a/pkg/server/routes.go b/pkg/server/routes.go index b68774b8..22b1c259 100644 --- a/pkg/server/routes.go +++ b/pkg/server/routes.go @@ -3,13 +3,15 @@ package server import ( "github.com/codeready-toolchain/registration-service/pkg/assets" "github.com/codeready-toolchain/registration-service/pkg/auth" - "github.com/gin-contrib/static" - "github.com/codeready-toolchain/registration-service/pkg/configuration" "github.com/codeready-toolchain/registration-service/pkg/controller" "github.com/codeready-toolchain/registration-service/pkg/middleware" + "github.com/gin-contrib/static" + "github.com/gin-gonic/gin" errs "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) // SetupRoutes registers handlers for various URL paths. @@ -21,6 +23,32 @@ func (srv *RegistrationServer) SetupRoutes(proxyPort string) error { return err } + inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "sandbox_promhttp_client_in_flight_requests", + Help: "A gauge of in-flight requests for the wrapped client.", + }) + + counter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "sandbox_promhttp_client_api_requests_total", + Help: "A counter for requests from the wrapped client.", + }, + []string{"code", "method", "path"}, + ) + + // histVec has no labels, making it a zero-dimensional ObserverVec. + histVec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "sandbox_promhttp_request_duration_seconds", + Help: "A histogram of request latencies.", + Buckets: prometheus.DefBuckets, + }, + []string{"code", "method", "path"}, + ) + + // Register all of the metrics in the standard registry. + prometheus.MustRegister(counter, histVec, inFlightGauge) + srv.routesSetup.Do(func() { // creating the controllers healthCheckCtrl := controller.NewHealthCheck(controller.NewHealthChecker(proxyPort)) @@ -29,6 +57,19 @@ func (srv *RegistrationServer) SetupRoutes(proxyPort string) error { signupCtrl := controller.NewSignup(srv.application) usernamesCtrl := controller.NewUsernames(srv.application) + // metrics endpoint (not mounted on `/api/v1`) + srv.router.GET("/metrics", gin.WrapH(promhttp.Handler())) + + // unsecured routes + unsecuredV1 := srv.router.Group("/api/v1") + unsecuredV1.Use( + middleware.InstrumentRoundTripperInFlight(inFlightGauge), + middleware.InstrumentRoundTripperCounter(counter), + middleware.InstrumentRoundTripperDuration(histVec)) + unsecuredV1.GET("/health", healthCheckCtrl.GetHandler) // TODO: move to root (`/`)? + unsecuredV1.GET("/authconfig", authConfigCtrl.GetHandler) + unsecuredV1.GET("/segment-write-key", analyticsCtrl.GetDevSpacesSegmentWriteKey) //expose the devspaces segment key + // create the auth middleware var authMiddleware *middleware.JWTMiddleware authMiddleware, err = middleware.NewAuthMiddleware() @@ -36,15 +77,13 @@ func (srv *RegistrationServer) SetupRoutes(proxyPort string) error { err = errs.Wrapf(err, "failed to init auth middleware") return } - - // unsecured routes - unsecuredV1 := srv.router.Group("/api/v1") - unsecuredV1.GET("/health", healthCheckCtrl.GetHandler) - unsecuredV1.GET("/authconfig", authConfigCtrl.GetHandler) - unsecuredV1.GET("/segment-write-key", analyticsCtrl.GetDevSpacesSegmentWriteKey) //expose the devspaces segment key // secured routes securedV1 := srv.router.Group("/api/v1") - securedV1.Use(authMiddleware.HandlerFunc()) + securedV1.Use( + middleware.InstrumentRoundTripperInFlight(inFlightGauge), + middleware.InstrumentRoundTripperCounter(counter), + middleware.InstrumentRoundTripperDuration(histVec), + authMiddleware.HandlerFunc()) securedV1.POST("/signup", signupCtrl.PostHandler) // requires a ctx body containing the country_code and phone_number securedV1.PUT("/signup/verification", signupCtrl.InitVerificationHandler)