From 35529b7c7efe0c996239e3f36d44dda50ad60d42 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 23 Aug 2023 17:04:41 +0300 Subject: [PATCH] Merging to release-1.8: TT-9873 Fix prometheus tracking path (#716) TT-9873 Fix prometheus tracking path (#716) * Fix prometheus tracking path * suggested fixes --- pumps/prometheus.go | 13 +++++++-- pumps/prometheus_test.go | 59 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/pumps/prometheus.go b/pumps/prometheus.go index b1b64adec..8613685a5 100644 --- a/pumps/prometheus.go +++ b/pumps/prometheus.go @@ -42,6 +42,9 @@ type PrometheusConf struct { AggregateObservations bool `json:"aggregate_observations" mapstructure:"aggregate_observations"` // Metrics to exclude from exposition. Currently, excludes only the base metrics. DisabledMetrics []string `json:"disabled_metrics" mapstructure:"disabled_metrics"` + // Specifies if it should expose aggregated metrics for all the endpoints. By default, `false` + // which means that all APIs endpoints will be counted as 'unknown' unless the API use track endpoint plugin. + TrackAllPaths bool `json:"track_all_paths" mapstructure:"track_all_paths"` // Custom Prometheus metrics. CustomMetrics CustomMetrics `json:"custom_metrics" mapstructure:"custom_metrics"` } @@ -91,8 +94,9 @@ type counterStruct struct { } const ( - counterType = "counter" - histogramType = "histogram" + counterType = "counter" + histogramType = "histogram" + prometheusUnknownPath = "unknown" ) var ( @@ -245,6 +249,11 @@ func (p *PrometheusPump) WriteData(ctx context.Context, data []interface{}) erro default: } record := item.(analytics.AnalyticsRecord) + + if !(p.conf.TrackAllPaths || record.TrackPath) { + record.Path = prometheusUnknownPath + } + // we loop through all the metrics available. for _, metric := range p.allMetrics { if metric.enabled { diff --git a/pumps/prometheus_test.go b/pumps/prometheus_test.go index 043fa5f55..383b1c61a 100644 --- a/pumps/prometheus_test.go +++ b/pumps/prometheus_test.go @@ -352,6 +352,7 @@ func TestPrometheusCounterMetric(t *testing.T) { analyticsRecords []analytics.AnalyticsRecord expectedMetricsAmount int expectedMetrics map[string]counterStruct + trackAllPaths bool }{ { testName: "HTTP status codes per API", @@ -375,7 +376,8 @@ func TestPrometheusCounterMetric(t *testing.T) { }, }, { - testName: "HTTP status codes per API path and method", + testName: "HTTP status codes per API path and method - trackign all paths", + trackAllPaths: true, metric: &PrometheusMetric{ Name: "tyk_http_status_per_path", Help: "HTTP status codes per API path and method", @@ -399,6 +401,57 @@ func TestPrometheusCounterMetric(t *testing.T) { "200--api_2--test--GET": {labelValues: []string{"200", "api_2", "test", "GET"}, count: 1}, }, }, + { + testName: "HTTP status codes per API path and method - tracking some paths", + trackAllPaths: false, + metric: &PrometheusMetric{ + Name: "tyk_http_status_per_path", + Help: "HTTP status codes per API path and method", + MetricType: counterType, + Labels: []string{"code", "api", "path", "method"}, + }, + analyticsRecords: []analytics.AnalyticsRecord{ + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET", TrackPath: true}, + {APIID: "api_1", ResponseCode: 500, Path: "test2", Method: "GET"}, + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET", TrackPath: true}, + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "POST", TrackPath: true}, + {APIID: "api_1", ResponseCode: 200, Path: "test2", Method: "GET"}, + {APIID: "api_2", ResponseCode: 200, Path: "test", Method: "GET"}, + }, + expectedMetricsAmount: 5, + expectedMetrics: map[string]counterStruct{ + "500--api_1--test--GET": {labelValues: []string{"500", "api_1", "test", "GET"}, count: 2}, + "500--api_1--test--POST": {labelValues: []string{"500", "api_1", "test", "POST"}, count: 1}, + "500--api_1--unknown--GET": {labelValues: []string{"500", "api_1", "unknown", "GET"}, count: 1}, + "200--api_1--unknown--GET": {labelValues: []string{"200", "api_1", "unknown", "GET"}, count: 1}, + "200--api_2--unknown--GET": {labelValues: []string{"200", "api_2", "unknown", "GET"}, count: 1}, + }, + }, + { + testName: "HTTP status codes per API path and method - not tracking paths", + trackAllPaths: false, + metric: &PrometheusMetric{ + Name: "tyk_http_status_per_path", + Help: "HTTP status codes per API path and method", + MetricType: counterType, + Labels: []string{"code", "api", "path", "method"}, + }, + analyticsRecords: []analytics.AnalyticsRecord{ + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET"}, + {APIID: "api_1", ResponseCode: 500, Path: "test2", Method: "GET"}, + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET"}, + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "POST"}, + {APIID: "api_1", ResponseCode: 200, Path: "test2", Method: "GET"}, + {APIID: "api_2", ResponseCode: 200, Path: "test", Method: "GET"}, + }, + expectedMetricsAmount: 4, + expectedMetrics: map[string]counterStruct{ + "500--api_1--unknown--GET": {labelValues: []string{"500", "api_1", "unknown", "GET"}, count: 3}, + "500--api_1--unknown--POST": {labelValues: []string{"500", "api_1", "unknown", "POST"}, count: 1}, + "200--api_1--unknown--GET": {labelValues: []string{"200", "api_1", "unknown", "GET"}, count: 1}, + "200--api_2--unknown--GET": {labelValues: []string{"200", "api_2", "unknown", "GET"}, count: 1}, + }, + }, { testName: "HTTP status codes per API key", metric: &PrometheusMetric{ @@ -474,6 +527,10 @@ func TestPrometheusCounterMetric(t *testing.T) { assert.Nil(t, err) defer prometheus.Unregister(tc.metric.counterVec) for _, record := range tc.analyticsRecords { + if !(tc.trackAllPaths || record.TrackPath) { + record.Path = "unknown" + } + labelValues := tc.metric.GetLabelsValues(record) assert.Equal(t, len(tc.metric.Labels), len(labelValues))