From a1fad74e91448d2fc17b931cbb22eca2bbbb5c87 Mon Sep 17 00:00:00 2001 From: Matias <83959431+mativm02@users.noreply.github.com> Date: Tue, 8 Aug 2023 09:28:05 -0300 Subject: [PATCH 1/3] [TT-9464] Updating storage to v1.0.8 (#698) * Updating storage to v1.0.7 * go mod tidy * Updating storage to v1.0.8 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1e6c52ea7..668bf524b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/TykTechnologies/gorpc v0.0.0-20210624160652-fe65bda0ccb9 github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20230320143102-7a16078ce517 github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632 - github.com/TykTechnologies/storage v1.0.7 + github.com/TykTechnologies/storage v1.0.8 github.com/aws/aws-sdk-go-v2 v1.16.14 github.com/aws/aws-sdk-go-v2/config v1.9.0 github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.9.0 diff --git a/go.sum b/go.sum index 9e21f1ae5..d2b7328b1 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20230320143102-7a16078ce517 github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20230320143102-7a16078ce517/go.mod h1:ZiFZcrue3+n2mHH+KLHRipbYVULkgy3Myko5S7IIs74= github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632 h1:T5NWziFusj8au5nxAqMMh/bZyX9CAyYnBkaMSsfH6BA= github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632/go.mod h1:UsPYgOFBpNzDXLEti7MKOwHLpVSqdzuNGkVFPspQmnQ= -github.com/TykTechnologies/storage v1.0.7 h1:lPkqSI5w15Eysh7JuOPCk/ajhgC4ibdHRhuiuoQQv2w= -github.com/TykTechnologies/storage v1.0.7/go.mod h1:+0S3KuNlLGBTMTSFREuZFm315zzXjuuCO4QSAPy+d3M= +github.com/TykTechnologies/storage v1.0.8 h1:MBs6hk5oLOmr2qK5/rl+dYO6iDMez6u3QkwOCL6K8n8= +github.com/TykTechnologies/storage v1.0.8/go.mod h1:+0S3KuNlLGBTMTSFREuZFm315zzXjuuCO4QSAPy+d3M= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= From 3bf1f8539f147adce0c5c879a852d78ae4bcabef Mon Sep 17 00:00:00 2001 From: Kofo Okesola Date: Wed, 23 Aug 2023 08:58:32 +0100 Subject: [PATCH 2/3] [TT-9855]: fix index creation error on graph sql pump creation (#715) * fix index creation error * fix ci * fix lint --- analytics/graph_record.go | 14 +++++++++++++- pumps/graph_sql.go | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/analytics/graph_record.go b/analytics/graph_record.go index a867e7660..160353713 100644 --- a/analytics/graph_record.go +++ b/analytics/graph_record.go @@ -20,6 +20,13 @@ import ( "github.com/TykTechnologies/storage/persistent/model" ) +// GraphSQLTableName should be defined before SQL migration is called on the GraphRecord +// the reason this approach is used to define the table name is due to gorm's inability to +// read values from the fields of the GraphRecord/AnalyticsRecord struct when it is migrating, due to that +// a single static value is going to be returned as TableName and it will be used as the prefix for index/relationship creation no matter the +// value passed to db.Table() +var GraphSQLTableName string + type GraphRecord struct { Types map[string][]string `gorm:"types"` @@ -32,8 +39,13 @@ type GraphRecord struct { HasErrors bool `gorm:"has_errors"` } +// TableName is used by both the sql orm and mongo driver the table name and collection name used for operations on this model +// the conditional return is to ensure the right value is used for both the sql and mongo operations func (g *GraphRecord) TableName() string { - return g.AnalyticsRecord.TableName() + if GraphSQLTableName == "" { + return g.AnalyticsRecord.TableName() + } + return GraphSQLTableName } // GetObjectID is a dummy function to satisfy the interface diff --git a/pumps/graph_sql.go b/pumps/graph_sql.go index 89b415ba2..0253f35eb 100644 --- a/pumps/graph_sql.go +++ b/pumps/graph_sql.go @@ -87,6 +87,7 @@ func (g *GraphSQLPump) Init(conf interface{}) error { if name := g.Conf.TableName; name != "" { g.tableName = name } + analytics.GraphSQLTableName = g.tableName if !g.Conf.TableSharding { if err := g.db.Table(g.tableName).AutoMigrate(&analytics.GraphRecord{}); err != nil { g.log.WithError(err).Error("error migrating graph analytics table") From 69f5f4a7d1b7edc436c9e943c404f5242cedc363 Mon Sep 17 00:00:00 2001 From: Tomas Buchaillot Date: Wed, 23 Aug 2023 15:54:53 +0200 Subject: [PATCH 3/3] TT-9873 Fix prometheus tracking path (#716) * Fix prometheus tracking path * suggested fixes --- pumps/prometheus.go | 13 +++++++-- pumps/prometheus_test.go | 59 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/pumps/prometheus.go b/pumps/prometheus.go index b1b64adec..8613685a5 100644 --- a/pumps/prometheus.go +++ b/pumps/prometheus.go @@ -42,6 +42,9 @@ type PrometheusConf struct { AggregateObservations bool `json:"aggregate_observations" mapstructure:"aggregate_observations"` // Metrics to exclude from exposition. Currently, excludes only the base metrics. DisabledMetrics []string `json:"disabled_metrics" mapstructure:"disabled_metrics"` + // Specifies if it should expose aggregated metrics for all the endpoints. By default, `false` + // which means that all APIs endpoints will be counted as 'unknown' unless the API use track endpoint plugin. + TrackAllPaths bool `json:"track_all_paths" mapstructure:"track_all_paths"` // Custom Prometheus metrics. CustomMetrics CustomMetrics `json:"custom_metrics" mapstructure:"custom_metrics"` } @@ -91,8 +94,9 @@ type counterStruct struct { } const ( - counterType = "counter" - histogramType = "histogram" + counterType = "counter" + histogramType = "histogram" + prometheusUnknownPath = "unknown" ) var ( @@ -245,6 +249,11 @@ func (p *PrometheusPump) WriteData(ctx context.Context, data []interface{}) erro default: } record := item.(analytics.AnalyticsRecord) + + if !(p.conf.TrackAllPaths || record.TrackPath) { + record.Path = prometheusUnknownPath + } + // we loop through all the metrics available. for _, metric := range p.allMetrics { if metric.enabled { diff --git a/pumps/prometheus_test.go b/pumps/prometheus_test.go index 043fa5f55..383b1c61a 100644 --- a/pumps/prometheus_test.go +++ b/pumps/prometheus_test.go @@ -352,6 +352,7 @@ func TestPrometheusCounterMetric(t *testing.T) { analyticsRecords []analytics.AnalyticsRecord expectedMetricsAmount int expectedMetrics map[string]counterStruct + trackAllPaths bool }{ { testName: "HTTP status codes per API", @@ -375,7 +376,8 @@ func TestPrometheusCounterMetric(t *testing.T) { }, }, { - testName: "HTTP status codes per API path and method", + testName: "HTTP status codes per API path and method - trackign all paths", + trackAllPaths: true, metric: &PrometheusMetric{ Name: "tyk_http_status_per_path", Help: "HTTP status codes per API path and method", @@ -399,6 +401,57 @@ func TestPrometheusCounterMetric(t *testing.T) { "200--api_2--test--GET": {labelValues: []string{"200", "api_2", "test", "GET"}, count: 1}, }, }, + { + testName: "HTTP status codes per API path and method - tracking some paths", + trackAllPaths: false, + metric: &PrometheusMetric{ + Name: "tyk_http_status_per_path", + Help: "HTTP status codes per API path and method", + MetricType: counterType, + Labels: []string{"code", "api", "path", "method"}, + }, + analyticsRecords: []analytics.AnalyticsRecord{ + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET", TrackPath: true}, + {APIID: "api_1", ResponseCode: 500, Path: "test2", Method: "GET"}, + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET", TrackPath: true}, + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "POST", TrackPath: true}, + {APIID: "api_1", ResponseCode: 200, Path: "test2", Method: "GET"}, + {APIID: "api_2", ResponseCode: 200, Path: "test", Method: "GET"}, + }, + expectedMetricsAmount: 5, + expectedMetrics: map[string]counterStruct{ + "500--api_1--test--GET": {labelValues: []string{"500", "api_1", "test", "GET"}, count: 2}, + "500--api_1--test--POST": {labelValues: []string{"500", "api_1", "test", "POST"}, count: 1}, + "500--api_1--unknown--GET": {labelValues: []string{"500", "api_1", "unknown", "GET"}, count: 1}, + "200--api_1--unknown--GET": {labelValues: []string{"200", "api_1", "unknown", "GET"}, count: 1}, + "200--api_2--unknown--GET": {labelValues: []string{"200", "api_2", "unknown", "GET"}, count: 1}, + }, + }, + { + testName: "HTTP status codes per API path and method - not tracking paths", + trackAllPaths: false, + metric: &PrometheusMetric{ + Name: "tyk_http_status_per_path", + Help: "HTTP status codes per API path and method", + MetricType: counterType, + Labels: []string{"code", "api", "path", "method"}, + }, + analyticsRecords: []analytics.AnalyticsRecord{ + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET"}, + {APIID: "api_1", ResponseCode: 500, Path: "test2", Method: "GET"}, + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "GET"}, + {APIID: "api_1", ResponseCode: 500, Path: "test", Method: "POST"}, + {APIID: "api_1", ResponseCode: 200, Path: "test2", Method: "GET"}, + {APIID: "api_2", ResponseCode: 200, Path: "test", Method: "GET"}, + }, + expectedMetricsAmount: 4, + expectedMetrics: map[string]counterStruct{ + "500--api_1--unknown--GET": {labelValues: []string{"500", "api_1", "unknown", "GET"}, count: 3}, + "500--api_1--unknown--POST": {labelValues: []string{"500", "api_1", "unknown", "POST"}, count: 1}, + "200--api_1--unknown--GET": {labelValues: []string{"200", "api_1", "unknown", "GET"}, count: 1}, + "200--api_2--unknown--GET": {labelValues: []string{"200", "api_2", "unknown", "GET"}, count: 1}, + }, + }, { testName: "HTTP status codes per API key", metric: &PrometheusMetric{ @@ -474,6 +527,10 @@ func TestPrometheusCounterMetric(t *testing.T) { assert.Nil(t, err) defer prometheus.Unregister(tc.metric.counterVec) for _, record := range tc.analyticsRecords { + if !(tc.trackAllPaths || record.TrackPath) { + record.Path = "unknown" + } + labelValues := tc.metric.GetLabelsValues(record) assert.Equal(t, len(tc.metric.Labels), len(labelValues))