Merge branch 'master' into fix/sqlaggregate-index

analytics/graph_record.go

@@ -20,6 +20,13 @@ import (
 	"github.com/TykTechnologies/storage/persistent/model"
 )
 
+// GraphSQLTableName should be defined before SQL migration is called on the GraphRecord
+// the reason this approach is used to define the table name is due to gorm's inability to
+// read values from the fields of the GraphRecord/AnalyticsRecord struct when it is migrating, due to that
+// a single static value is going to be returned as TableName and it will be used as the prefix for index/relationship creation no matter the
+// value passed to db.Table()
+var GraphSQLTableName string
+
 type GraphRecord struct {
 	Types map[string][]string `gorm:"types"`
 
@@ -32,8 +39,13 @@ type GraphRecord struct {
 	HasErrors     bool         `gorm:"has_errors"`
 }
 
+// TableName is used by both the sql orm and mongo driver the table name and collection name used for operations on this model
+// the conditional return is to ensure the right value is used for both the sql and mongo operations
 func (g *GraphRecord) TableName() string {
-	return g.AnalyticsRecord.TableName()
+	if GraphSQLTableName == "" {
+		return g.AnalyticsRecord.TableName()
+	}
+	return GraphSQLTableName
 }
 
 // GetObjectID is a dummy function to satisfy the interface

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/TykTechnologies/gorpc v0.0.0-20210624160652-fe65bda0ccb9
 	github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20230320143102-7a16078ce517
 	github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632
-	github.com/TykTechnologies/storage v1.0.7
+	github.com/TykTechnologies/storage v1.0.8
 	github.com/aws/aws-sdk-go-v2 v1.16.14
 	github.com/aws/aws-sdk-go-v2/config v1.9.0
 	github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.9.0

go.sum

@@ -53,8 +53,8 @@ github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20230320143102-7a16078ce517
 github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20230320143102-7a16078ce517/go.mod h1:ZiFZcrue3+n2mHH+KLHRipbYVULkgy3Myko5S7IIs74=
 github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632 h1:T5NWziFusj8au5nxAqMMh/bZyX9CAyYnBkaMSsfH6BA=
 github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632/go.mod h1:UsPYgOFBpNzDXLEti7MKOwHLpVSqdzuNGkVFPspQmnQ=
-github.com/TykTechnologies/storage v1.0.7 h1:lPkqSI5w15Eysh7JuOPCk/ajhgC4ibdHRhuiuoQQv2w=
-github.com/TykTechnologies/storage v1.0.7/go.mod h1:+0S3KuNlLGBTMTSFREuZFm315zzXjuuCO4QSAPy+d3M=
+github.com/TykTechnologies/storage v1.0.8 h1:MBs6hk5oLOmr2qK5/rl+dYO6iDMez6u3QkwOCL6K8n8=
+github.com/TykTechnologies/storage v1.0.8/go.mod h1:+0S3KuNlLGBTMTSFREuZFm315zzXjuuCO4QSAPy+d3M=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
 github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=

pumps/graph_sql.go

@@ -87,6 +87,7 @@ func (g *GraphSQLPump) Init(conf interface{}) error {
 	if name := g.Conf.TableName; name != "" {
 		g.tableName = name
 	}
+	analytics.GraphSQLTableName = g.tableName
 	if !g.Conf.TableSharding {
 		if err := g.db.Table(g.tableName).AutoMigrate(&analytics.GraphRecord{}); err != nil {
 			g.log.WithError(err).Error("error migrating graph analytics table")

pumps/prometheus.go

@@ -42,6 +42,9 @@ type PrometheusConf struct {
 	AggregateObservations bool `json:"aggregate_observations" mapstructure:"aggregate_observations"`
 	// Metrics to exclude from exposition. Currently, excludes only the base metrics.
 	DisabledMetrics []string `json:"disabled_metrics" mapstructure:"disabled_metrics"`
+	// Specifies if it should expose aggregated metrics for all the endpoints. By default, `false`
+	// which means that all APIs endpoints will be counted as 'unknown' unless the API use track endpoint plugin.
+	TrackAllPaths bool `json:"track_all_paths" mapstructure:"track_all_paths"`
 	// Custom Prometheus metrics.
 	CustomMetrics CustomMetrics `json:"custom_metrics" mapstructure:"custom_metrics"`
 }
@@ -91,8 +94,9 @@ type counterStruct struct {
 }
 
 const (
-	counterType   = "counter"
-	histogramType = "histogram"
+	counterType           = "counter"
+	histogramType         = "histogram"
+	prometheusUnknownPath = "unknown"
 )
 
 var (
@@ -245,6 +249,11 @@ func (p *PrometheusPump) WriteData(ctx context.Context, data []interface{}) erro
 		default:
 		}
 		record := item.(analytics.AnalyticsRecord)
+
+		if !(p.conf.TrackAllPaths || record.TrackPath) {
+			record.Path = prometheusUnknownPath
+		}
+
 		// we loop through all the metrics available.
 		for _, metric := range p.allMetrics {
 			if metric.enabled {

pumps/prometheus_test.go

@@ -352,6 +352,7 @@ func TestPrometheusCounterMetric(t *testing.T) {
 		analyticsRecords      []analytics.AnalyticsRecord
 		expectedMetricsAmount int
 		expectedMetrics       map[string]counterStruct
+		trackAllPaths         bool
 	}{
 		{
 			testName: "HTTP status codes per API",
@@ -375,7 +376,8 @@ func TestPrometheusCounterMetric(t *testing.T) {
 			},
 		},
 		{
-			testName: "HTTP status codes per API path and method",
+			testName:      "HTTP status codes per API path and method - trackign all paths",
+			trackAllPaths: true,
 			metric: &PrometheusMetric{
 				Name:       "tyk_http_status_per_path",
 				Help:       "HTTP status codes per API path and method",
@@ -399,6 +401,57 @@ func TestPrometheusCounterMetric(t *testing.T) {
 				"200--api_2--test--GET":  {labelValues: []string{"200", "api_2", "test", "GET"}, count: 1},
 			},
 		},
+		{
+			testName:      "HTTP status codes per API path and method - tracking some paths",
+			trackAllPaths: false,
+			metric: &PrometheusMetric{
+				Name:       "tyk_http_status_per_path",
+				Help:       "HTTP status codes per API path and method",
+				MetricType: counterType,
+				Labels:     []string{"code", "api", "path", "method"},
+			},
+			analyticsRecords: []analytics.AnalyticsRecord{
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET", TrackPath: true},
+				{APIID: "api_1", ResponseCode: 500, Path: "test2", Method: "GET"},
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET", TrackPath: true},
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "POST", TrackPath: true},
+				{APIID: "api_1", ResponseCode: 200, Path: "test2", Method: "GET"},
+				{APIID: "api_2", ResponseCode: 200, Path: "test", Method: "GET"},
+			},
+			expectedMetricsAmount: 5,
+			expectedMetrics: map[string]counterStruct{
+				"500--api_1--test--GET":    {labelValues: []string{"500", "api_1", "test", "GET"}, count: 2},
+				"500--api_1--test--POST":   {labelValues: []string{"500", "api_1", "test", "POST"}, count: 1},
+				"500--api_1--unknown--GET": {labelValues: []string{"500", "api_1", "unknown", "GET"}, count: 1},
+				"200--api_1--unknown--GET": {labelValues: []string{"200", "api_1", "unknown", "GET"}, count: 1},
+				"200--api_2--unknown--GET": {labelValues: []string{"200", "api_2", "unknown", "GET"}, count: 1},
+			},
+		},
+		{
+			testName:      "HTTP status codes per API path and method - not tracking paths",
+			trackAllPaths: false,
+			metric: &PrometheusMetric{
+				Name:       "tyk_http_status_per_path",
+				Help:       "HTTP status codes per API path and method",
+				MetricType: counterType,
+				Labels:     []string{"code", "api", "path", "method"},
+			},
+			analyticsRecords: []analytics.AnalyticsRecord{
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET"},
+				{APIID: "api_1", ResponseCode: 500, Path: "test2", Method: "GET"},
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET"},
+				{APIID: "api_1", ResponseCode: 500, Path: "test", Method: "POST"},
+				{APIID: "api_1", ResponseCode: 200, Path: "test2", Method: "GET"},
+				{APIID: "api_2", ResponseCode: 200, Path: "test", Method: "GET"},
+			},
+			expectedMetricsAmount: 4,
+			expectedMetrics: map[string]counterStruct{
+				"500--api_1--unknown--GET":  {labelValues: []string{"500", "api_1", "unknown", "GET"}, count: 3},
+				"500--api_1--unknown--POST": {labelValues: []string{"500", "api_1", "unknown", "POST"}, count: 1},
+				"200--api_1--unknown--GET":  {labelValues: []string{"200", "api_1", "unknown", "GET"}, count: 1},
+				"200--api_2--unknown--GET":  {labelValues: []string{"200", "api_2", "unknown", "GET"}, count: 1},
+			},
+		},
 		{
 			testName: "HTTP status codes per API key",
 			metric: &PrometheusMetric{
@@ -474,6 +527,10 @@ func TestPrometheusCounterMetric(t *testing.T) {
 			assert.Nil(t, err)
 			defer prometheus.Unregister(tc.metric.counterVec)
 			for _, record := range tc.analyticsRecords {
+				if !(tc.trackAllPaths || record.TrackPath) {
+					record.Path = "unknown"
+				}
+
 				labelValues := tc.metric.GetLabelsValues(record)
 				assert.Equal(t, len(tc.metric.Labels), len(labelValues))