Merging to release-1.8: TT-9873 Fix prometheus tracking path (#716)

TT-9873 Fix prometheus tracking path (#716) * Fix prometheus tracking path * suggested fixes
TykTechnologies · Aug 23, 2023 · 35529b7 · 35529b7
1 parent c470fd6
commit 35529b7
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 3 deletions.
diff --git a/pumps/prometheus.go b/pumps/prometheus.go
@@ -42,6 +42,9 @@ type PrometheusConf struct {
 	AggregateObservations bool `json:"aggregate_observations" mapstructure:"aggregate_observations"`
 	// Metrics to exclude from exposition. Currently, excludes only the base metrics.
 	DisabledMetrics []string `json:"disabled_metrics" mapstructure:"disabled_metrics"`
+	// Specifies if it should expose aggregated metrics for all the endpoints. By default, `false`
+	// which means that all APIs endpoints will be counted as 'unknown' unless the API use track endpoint plugin.
+	TrackAllPaths bool `json:"track_all_paths" mapstructure:"track_all_paths"`
 	// Custom Prometheus metrics.
 	CustomMetrics CustomMetrics `json:"custom_metrics" mapstructure:"custom_metrics"`
 }
@@ -91,8 +94,9 @@ type counterStruct struct {
 }
 
 const (
-	counterType   = "counter"
-	histogramType = "histogram"
+	counterType           = "counter"
+	histogramType         = "histogram"
+	prometheusUnknownPath = "unknown"
 )
 
 var (
@@ -245,6 +249,11 @@ func (p *PrometheusPump) WriteData(ctx context.Context, data []interface{}) erro
 		default:
 		}
 		record := item.(analytics.AnalyticsRecord)
+
+		if !(p.conf.TrackAllPaths || record.TrackPath) {
+			record.Path = prometheusUnknownPath
+		}
+
 		// we loop through all the metrics available.
 		for _, metric := range p.allMetrics {
 			if metric.enabled {

diff --git a/pumps/prometheus_test.go b/pumps/prometheus_test.go
@@ -352,6 +352,7 @@ func TestPrometheusCounterMetric(t *testing.T) {
 		analyticsRecords      []analytics.AnalyticsRecord
 		expectedMetricsAmount int
 		expectedMetrics       map[string]counterStruct
+		trackAllPaths         bool
 	}{
 		{
 			testName: "HTTP status codes per API",
@@ -375,7 +376,8 @@ func TestPrometheusCounterMetric(t *testing.T) {
 			},
 		},
 		{
-			testName: "HTTP status codes per API path and method",
+			testName:      "HTTP status codes per API path and method - trackign all paths",
+			trackAllPaths: true,
 			metric: &PrometheusMetric{
 				Name:       "tyk_http_status_per_path",
 				Help:       "HTTP status codes per API path and method",
@@ -399,6 +401,57 @@ func TestPrometheusCounterMetric(t *testing.T) {
 				"200--api_2--test--GET":  {labelValues: []string{"200", "api_2", "test", "GET"}, count: 1},
 			},
 		},
+		{
+			testName:      "HTTP status codes per API path and method - tracking some paths",
+			trackAllPaths: false,
+			metric: &PrometheusMetric{
+				Name:       "tyk_http_status_per_path",
+				Help:       "HTTP status codes per API path and method",
+				MetricType: counterType,
+				Labels:     []string{"code", "api", "path", "method"},
+			},
+			analyticsRecords: []analytics.AnalyticsRecord{
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET", TrackPath: true},
+				{APIID: "api_1", ResponseCode: 500, Path: "test2", Method: "GET"},
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET", TrackPath: true},
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "POST", TrackPath: true},
+				{APIID: "api_1", ResponseCode: 200, Path: "test2", Method: "GET"},
+				{APIID: "api_2", ResponseCode: 200, Path: "test", Method: "GET"},
+			},
+			expectedMetricsAmount: 5,
+			expectedMetrics: map[string]counterStruct{
+				"500--api_1--test--GET":    {labelValues: []string{"500", "api_1", "test", "GET"}, count: 2},
+				"500--api_1--test--POST":   {labelValues: []string{"500", "api_1", "test", "POST"}, count: 1},
+				"500--api_1--unknown--GET": {labelValues: []string{"500", "api_1", "unknown", "GET"}, count: 1},
+				"200--api_1--unknown--GET": {labelValues: []string{"200", "api_1", "unknown", "GET"}, count: 1},
+				"200--api_2--unknown--GET": {labelValues: []string{"200", "api_2", "unknown", "GET"}, count: 1},
+			},
+		},
+		{
+			testName:      "HTTP status codes per API path and method - not tracking paths",
+			trackAllPaths: false,
+			metric: &PrometheusMetric{
+				Name:       "tyk_http_status_per_path",
+				Help:       "HTTP status codes per API path and method",
+				MetricType: counterType,
+				Labels:     []string{"code", "api", "path", "method"},
+			},
+			analyticsRecords: []analytics.AnalyticsRecord{
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET"},
+				{APIID: "api_1", ResponseCode: 500, Path: "test2", Method: "GET"},
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET"},
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "POST"},
+				{APIID: "api_1", ResponseCode: 200, Path: "test2", Method: "GET"},
+				{APIID: "api_2", ResponseCode: 200, Path: "test", Method: "GET"},
+			},
+			expectedMetricsAmount: 4,
+			expectedMetrics: map[string]counterStruct{
+				"500--api_1--unknown--GET":  {labelValues: []string{"500", "api_1", "unknown", "GET"}, count: 3},
+				"500--api_1--unknown--POST": {labelValues: []string{"500", "api_1", "unknown", "POST"}, count: 1},
+				"200--api_1--unknown--GET":  {labelValues: []string{"200", "api_1", "unknown", "GET"}, count: 1},
+				"200--api_2--unknown--GET":  {labelValues: []string{"200", "api_2", "unknown", "GET"}, count: 1},
+			},
+		},
 		{
 			testName: "HTTP status codes per API key",
 			metric: &PrometheusMetric{
@@ -474,6 +527,10 @@ func TestPrometheusCounterMetric(t *testing.T) {
 			assert.Nil(t, err)
 			defer prometheus.Unregister(tc.metric.counterVec)
 			for _, record := range tc.analyticsRecords {
+				if !(tc.trackAllPaths || record.TrackPath) {
+					record.Path = "unknown"
+				}
+
 				labelValues := tc.metric.GetLabelsValues(record)
 				assert.Equal(t, len(tc.metric.Labels), len(labelValues))