[TT-9467]: Added graphstats processing to recorded analytics (#5716)

<!-- Provide a general summary of your changes in the Title above -->

## Description
This PR makes use of the incoming changes to the pump in this
[PR](TykTechnologies/tyk-pump#736) to allow the
graph pumps to record analytics without the need to enable detailed
recording

[TT-9467](https://tyktech.atlassian.net/browse/TT-9476)
<!-- Describe your changes in detail -->

## Related Issue

<!-- This project only accepts pull requests related to open issues. -->
<!-- If suggesting a new feature or change, please discuss it in an
issue first. -->
<!-- If fixing a bug, there should be an issue describing it with steps
to reproduce. -->
<!-- OSS: Please link to the issue here. Tyk: please create/link the
JIRA ticket. -->

## Motivation and Context

<!-- Why is this change required? What problem does it solve? -->

## How This Has Been Tested

<!-- Please describe in detail how you tested your changes -->
<!-- Include details of your testing environment, and the tests -->
<!-- you ran to see how your change affects other areas of the code,
etc. -->
<!-- This information is helpful for reviewers and QA. -->

## Screenshots (if appropriate)

## Types of changes

<!-- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Refactoring or add test (improvements in base code or adds test
coverage to functionality)

## Checklist

<!-- Go over all the following points, and put an `x` in all the boxes
that apply -->
<!-- If there are no documentation updates required, mark the item as
checked. -->
<!-- Raise up any additional concerns not covered by the checklist. -->

- [ ] I ensured that the documentation is up to date
- [ ] I explained why this PR updates go.mod in detail with reasoning
why it's required
- [ ] I would like a code coverage CI quality gate exception and have
explained why


[TT-9467]:
https://tyktech.atlassian.net/browse/TT-9467?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ

gateway/handler_error.go

@@ -259,24 +259,6 @@ func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMs
 		if len(e.Spec.Tags) > 0 {
 			tags = append(tags, e.Spec.Tags...)
 		}
-
-		rawRequest := ""
-		rawResponse := ""
-
-		if recordDetail(r, e.Spec) {
-
-			// Get the wire format representation
-
-			var wireFormatReq bytes.Buffer
-			r.Write(&wireFormatReq)
-			rawRequest = base64.StdEncoding.EncodeToString(wireFormatReq.Bytes())
-
-			var wireFormatRes bytes.Buffer
-			response.Write(&wireFormatRes)
-			rawResponse = base64.StdEncoding.EncodeToString(wireFormatRes.Bytes())
-
-		}
-
 		trackEP := false
 		trackedPath := r.URL.Path
 
@@ -312,8 +294,6 @@ func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMs
 			OauthID:       oauthClientID,
 			RequestTime:   0,
 			Latency:       analytics.Latency{},
-			RawRequest:    rawRequest,
-			RawResponse:   rawResponse,
 			IPAddress:     ip,
 			Geo:           analytics.GeoData{},
 			Network:       analytics.NetworkStats{},
@@ -322,6 +302,26 @@ func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMs
 			TrackPath:     trackEP,
 			ExpireAt:      t,
 		}
+		recordGraphDetails(&record, r, response, e.Spec)
+
+		rawRequest := ""
+		rawResponse := ""
+		if recordDetail(r, e.Spec) {
+
+			// Get the wire format representation
+
+			var wireFormatReq bytes.Buffer
+			r.Write(&wireFormatReq)
+			rawRequest = base64.StdEncoding.EncodeToString(wireFormatReq.Bytes())
+
+			var wireFormatRes bytes.Buffer
+			response.Write(&wireFormatRes)
+			rawResponse = base64.StdEncoding.EncodeToString(wireFormatRes.Bytes())
+
+		}
+
+		record.RawRequest = rawRequest
+		record.RawResponse = rawResponse
 
 		if e.Spec.GlobalConfig.AnalyticsConfig.EnableGeoIP {
 			record.GetGeo(ip, e.Gw.Analytics.GeoIPDB)

gateway/handler_success.go

@@ -11,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/TykTechnologies/tyk/internal/graphql"
+
 	"github.com/TykTechnologies/tyk/apidef"
 	"github.com/TykTechnologies/tyk/internal/httputil"
 
@@ -119,6 +121,48 @@ func getSessionTags(session *user.SessionState) []string {
 	return tags
 }
 
+func recordGraphDetails(rec *analytics.AnalyticsRecord, r *http.Request, resp *http.Response, spec *APISpec) {
+	if !spec.GraphQL.Enabled || spec.GraphQL.ExecutionMode == apidef.GraphQLExecutionModeSubgraph {
+		return
+	}
+	logger := log.WithField("location", "recordGraphDetails")
+	if r.Body == nil {
+		return
+	}
+	body, err := io.ReadAll(r.Body)
+	defer func() {
+		_ = r.Body.Close()
+		r.Body = io.NopCloser(bytes.NewBuffer(body))
+	}()
+	if err != nil {
+		logger.WithError(err).Error("error recording graph analytics")
+		return
+	}
+	var (
+		respBody []byte
+	)
+	if resp.Body != nil {
+		httputil.RemoveResponseTransferEncoding(resp, "chunked")
+		respBody, err = io.ReadAll(resp.Body)
+		defer func() {
+			_ = resp.Body.Close()
+			resp.Body = respBodyReader(r, resp)
+		}()
+		if err != nil {
+			logger.WithError(err).Error("error recording graph analytics")
+			return
+		}
+	}
+
+	extractor := graphql.NewGraphStatsExtractor()
+	stats, err := extractor.ExtractStats(string(body), string(respBody), spec.GraphQL.Schema)
+	if err != nil {
+		logger.WithError(err).Error("error recording graph analytics")
+		return
+	}
+	rec.GraphQLStats = stats
+}
+
 func (s *SuccessHandler) RecordHit(r *http.Request, timing analytics.Latency, code int, responseCopy *http.Response) {
 
 	if s.Spec.DoNotTrack || ctxGetDoNotTrack(r) {
@@ -177,7 +221,7 @@ func (s *SuccessHandler) RecordHit(r *http.Request, timing analytics.Latency, co
 				// we need to delete the chunked transfer encoding header to avoid malformed body in our rawResponse
 				httputil.RemoveResponseTransferEncoding(responseCopy, "chunked")
 
-				contents, err := ioutil.ReadAll(responseCopy.Body)
+				responseContent, err := io.ReadAll(responseCopy.Body)
 				if err != nil {
 					log.Error("Couldn't read response body", err)
 				}
@@ -187,7 +231,7 @@ func (s *SuccessHandler) RecordHit(r *http.Request, timing analytics.Latency, co
 				// Get the wire format representation
 				var wireFormatRes bytes.Buffer
 				responseCopy.Write(&wireFormatRes)
-				responseCopy.Body = ioutil.NopCloser(bytes.NewBuffer(contents))
+				responseCopy.Body = ioutil.NopCloser(bytes.NewBuffer(responseContent))
 				rawResponse = base64.StdEncoding.EncodeToString(wireFormatRes.Bytes())
 			}
 		}
@@ -240,6 +284,7 @@ func (s *SuccessHandler) RecordHit(r *http.Request, timing analytics.Latency, co
 			record.GetGeo(ip, s.Gw.Analytics.GeoIPDB)
 		}
 
+		recordGraphDetails(&record, r, responseCopy, s.Spec)
 		// skip tagging subgraph requests for graphpump, it only handles generated supergraph requests
 		if s.Spec.GraphQL.Enabled && s.Spec.GraphQL.ExecutionMode != apidef.GraphQLExecutionModeSubgraph {
 			record.Tags = append(record.Tags, "tyk-graph-analytics")
@@ -270,7 +315,6 @@ func (s *SuccessHandler) RecordHit(r *http.Request, timing analytics.Latency, co
 		}
 
 		err := s.Gw.Analytics.RecordHit(&record)
-
 		if err != nil {
 			log.WithError(err).Error("could not store analytic record")
 		}

gateway/handler_success_test.go

@@ -111,6 +111,131 @@ func TestRecordDetail(t *testing.T) {
 	}
 }
 
+func TestAnalyticRecord_GraphStats(t *testing.T) {
+
+	apiDef := BuildAPI(func(spec *APISpec) {
+		spec.Name = "graphql API"
+		spec.APIID = "graphql-api"
+		spec.Proxy.TargetURL = testGraphQLProxyUpstream
+		spec.Proxy.ListenPath = "/"
+		spec.GraphQL = apidef.GraphQLConfig{
+			Enabled:       true,
+			ExecutionMode: apidef.GraphQLExecutionModeProxyOnly,
+			Version:       apidef.GraphQLConfigVersion2,
+			Schema:        gqlProxyUpstreamSchema,
+		}
+	})[0]
+
+	testCases := []struct {
+		name      string
+		code      int
+		request   graphql.Request
+		checkFunc func(*testing.T, *analytics.AnalyticsRecord)
+		reloadAPI func(APISpec) *APISpec
+	}{
+		{
+			name: "successfully generate stats",
+			code: http.StatusOK,
+			request: graphql.Request{
+				Query: `{ hello(name: "World") httpMethod }`,
+			},
+			checkFunc: func(t *testing.T, record *analytics.AnalyticsRecord) {
+				assert.True(t, record.GraphQLStats.IsGraphQL)
+				assert.False(t, record.GraphQLStats.HasErrors)
+				assert.Equal(t, []string{"hello", "httpMethod"}, record.GraphQLStats.RootFields)
+				assert.Equal(t, map[string][]string{}, record.GraphQLStats.Types)
+				assert.Equal(t, analytics.OperationQuery, record.GraphQLStats.OperationType)
+			},
+		},
+		{
+			name: "should have variables",
+			code: http.StatusOK,
+			request: graphql.Request{
+				Query:     `{ hello(name: "World") httpMethod }`,
+				Variables: []byte(`{"in":"hello"}`),
+			},
+			checkFunc: func(t *testing.T, record *analytics.AnalyticsRecord) {
+				assert.True(t, record.GraphQLStats.IsGraphQL)
+				assert.False(t, record.GraphQLStats.HasErrors)
+				assert.Equal(t, []string{"hello", "httpMethod"}, record.GraphQLStats.RootFields)
+				assert.Equal(t, map[string][]string{}, record.GraphQLStats.Types)
+				assert.Equal(t, analytics.OperationQuery, record.GraphQLStats.OperationType)
+				assert.Equal(t, `{"in":"hello"}`, record.GraphQLStats.Variables)
+			},
+		},
+		{
+			name: "should read response and error response request with detailed recording",
+			code: http.StatusInternalServerError,
+			request: graphql.Request{
+				Query:     `{ hello(name: "World") httpMethod }`,
+				Variables: []byte(`{"in":"hello"}`),
+			},
+			reloadAPI: func(spec APISpec) *APISpec {
+				spec.Proxy.TargetURL = testGraphQLProxyUpstreamError
+				spec.EnableDetailedRecording = true
+				return &spec
+			},
+			checkFunc: func(t *testing.T, record *analytics.AnalyticsRecord) {
+				assert.True(t, record.GraphQLStats.IsGraphQL)
+				assert.True(t, record.GraphQLStats.HasErrors)
+				assert.Equal(t, []string{"hello", "httpMethod"}, record.GraphQLStats.RootFields)
+				assert.Equal(t, map[string][]string{}, record.GraphQLStats.Types)
+				assert.Equal(t, analytics.OperationQuery, record.GraphQLStats.OperationType)
+				assert.Equal(t, `{"in":"hello"}`, record.GraphQLStats.Variables)
+				assert.Equal(t, []analytics.GraphError{
+					{Message: "unable to resolve"},
+				}, record.GraphQLStats.Errors)
+			},
+		},
+		{
+			name: "should read response request without detailed recording",
+			code: http.StatusInternalServerError,
+			request: graphql.Request{
+				Query:     `{ hello(name: "World") httpMethod }`,
+				Variables: []byte(`{"in":"hello"}`),
+			},
+			reloadAPI: func(spec APISpec) *APISpec {
+				spec.Proxy.TargetURL = testGraphQLProxyUpstreamError
+				return &spec
+			},
+			checkFunc: func(t *testing.T, record *analytics.AnalyticsRecord) {
+				assert.True(t, record.GraphQLStats.IsGraphQL)
+				assert.True(t, record.GraphQLStats.HasErrors)
+				assert.Equal(t, []string{"hello", "httpMethod"}, record.GraphQLStats.RootFields)
+				assert.Equal(t, map[string][]string{}, record.GraphQLStats.Types)
+				assert.Equal(t, analytics.OperationQuery, record.GraphQLStats.OperationType)
+				assert.Equal(t, `{"in":"hello"}`, record.GraphQLStats.Variables)
+				assert.Equal(t, []analytics.GraphError{
+					{Message: "unable to resolve"},
+				}, record.GraphQLStats.Errors)
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			spec := apiDef
+			if tc.reloadAPI != nil {
+				spec = tc.reloadAPI(*apiDef)
+			}
+
+			ts := StartTest(nil)
+			defer ts.Close()
+			ts.Gw.LoadAPI(spec)
+			ts.Gw.Analytics.mockEnabled = true
+			ts.Gw.Analytics.mockRecordHit = func(record *analytics.AnalyticsRecord) {
+				tc.checkFunc(t, record)
+			}
+			_, err := ts.Run(t, test.TestCase{
+				Data:   tc.request,
+				Method: http.MethodPost,
+				Code:   tc.code,
+			})
+			assert.NoError(t, err)
+		})
+	}
+}
+
 func TestAnalyticsIgnoreSubgraph(t *testing.T) {
 	ts := StartTest(nil)
 	defer ts.Close()
@@ -136,6 +261,7 @@ func TestAnalyticsIgnoreSubgraph(t *testing.T) {
 		spec.Name = "supergraph"
 		spec.APIID = "supergraph"
 		spec.Proxy.ListenPath = "/supergraph"
+		spec.EnableDetailedRecording = true
 		spec.GraphQL = apidef.GraphQLConfig{
 			Enabled:       true,
 			ExecutionMode: apidef.GraphQLExecutionModeSupergraph,
@@ -172,6 +298,7 @@ func TestAnalyticsIgnoreSubgraph(t *testing.T) {
 		if record.ApiSchema != "" && found {
 			t.Error("subgraph request should not tagged or have schema")
 		}
+		assert.False(t, record.GraphQLStats.IsGraphQL)
 	}
 
 	_, err := ts.Run(t,

gateway/reverse_proxy.go

@@ -515,7 +515,12 @@ var hopHeaders = []string{
 func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) ProxyResponse {
 	startTime := time.Now()
 	p.logger.WithField("ts", startTime.UnixNano()).Debug("Started")
-	resp := p.WrappedServeHTTP(rw, req, recordDetail(req, p.TykAPISpec))
+	var resp ProxyResponse
+	if IsGrpcStreaming(req) {
+		resp = p.WrappedServeHTTP(rw, req, false)
+	} else {
+		resp = p.WrappedServeHTTP(rw, req, true)
+	}
 
 	finishTime := time.Since(startTime)
 	p.logger.WithField("ns", finishTime.Nanoseconds()).Debug("Finished")

go.mod

@@ -13,12 +13,12 @@ require (
 	github.com/TykTechnologies/gojsonschema v0.0.0-20170222154038-dcb3e4bb7990
 	github.com/TykTechnologies/gorpc v0.0.0-20210624160652-fe65bda0ccb9
 	github.com/TykTechnologies/goverify v0.0.0-20220808203004-1486f89e7708
-	github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20231017082933-70819d7e4c9b
+	github.com/TykTechnologies/graphql-go-tools v1.6.2-0.20231106100746-27618ae672d3
 	github.com/TykTechnologies/leakybucket v0.0.0-20170301023702-71692c943e3c
 	github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632
 	github.com/TykTechnologies/openid2go v0.1.2
-	github.com/TykTechnologies/storage v1.0.5
-	github.com/TykTechnologies/tyk-pump v1.8.0-rc4
+	github.com/TykTechnologies/storage v1.0.8
+	github.com/TykTechnologies/tyk-pump v1.8.1-rc1.0.20231030094653-9984a1ee29ee
 	github.com/akutz/memconn v0.1.0
 	github.com/bshuster-repo/logrus-logstash-hook v1.1.0
 	github.com/buger/jsonparser v1.1.1
@@ -30,7 +30,7 @@ require (
 	github.com/getkin/kin-openapi v0.115.0
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0
-	github.com/gofrs/uuid v3.3.0+incompatible
+	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/golang-jwt/jwt/v4 v4.4.2
 	github.com/golang/mock v1.4.4
 	github.com/golang/protobuf v1.5.3
@@ -53,7 +53,7 @@ require (
 	github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e // test
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/openzipkin/zipkin-go v0.2.2
-	github.com/oschwald/maxminddb-golang v1.5.0
+	github.com/oschwald/maxminddb-golang v1.11.0
 	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/pires/go-proxyproto v0.7.0
 	github.com/pmylund/go-cache v2.1.0+incompatible
@@ -82,6 +82,7 @@ require (
 require (
 	github.com/TykTechnologies/kin-openapi v0.90.0
 	github.com/TykTechnologies/opentelemetry v0.0.20
+	github.com/google/go-cmp v0.5.9
 	go.opentelemetry.io/otel v1.19.0
 	go.opentelemetry.io/otel/trace v1.19.0
 )
@@ -94,7 +95,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
-	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
 	github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
 	github.com/asyncapi/converter-go v0.0.0-20190802111537-d8459b2bd403 // indirect
@@ -111,6 +112,7 @@ require (
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/frankban/quicktest v1.14.6 // indirect
 	github.com/getsentry/raven-go v0.2.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
@@ -122,6 +124,7 @@ require (
 	github.com/gobwas/pool v0.2.0 // indirect
 	github.com/gobwas/ws v1.0.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/btree v1.0.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
@@ -200,7 +203,7 @@ require (
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/square/go-jose.v2 v2.3.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gorm.io/gorm v1.21.10 // indirect
+	gorm.io/gorm v1.21.16 // indirect
 )
 
 //replace github.com/TykTechnologies/graphql-go-tools => ../graphql-go-tools