diff --git a/CHANGELOG.md b/CHANGELOG.md index dbb6dc82745..39a9e91106f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## main / unreleased -* [FEATURE] tempo-cli: support dropping multiple traces in a single operation [#4266](https://github.com/grafana/tempo/pull/4266) (@ndk) + * [CHANGE] **BREAKING CHANGE** Add maximum spans per span set. Users can set `max_spans_per_span_set` to 0 to obtain the old behavior. [#4275](https://github.com/grafana/tempo/pull/4383) (@carles-grafana) * [CHANGE] slo: include request cancellations within SLO [#4355] (https://github.com/grafana/tempo/pull/4355) (@electron0zero) request cancellations are exposed under `result` label in `tempo_query_frontend_queries_total` and `tempo_query_frontend_queries_within_slo_total` with `completed` or `canceled` values to differentiate between completed and canceled requests. @@ -22,10 +22,12 @@ * [CHANGE] Return 422 for TRACE_TOO_LARGE queries [#4160](https://github.com/grafana/tempo/pull/4160) (@zalegrala) * [CHANGE] Upgrade OTEL sdk to reduce allocs [#4243](https://github.com/grafana/tempo/pull/4243) (@joe-elliott) * [CHANGE] Tighten file permissions [#4251](https://github.com/grafana/tempo/pull/4251) (@zalegrala) +* [FEATURE] tempo-cli: support dropping multiple traces in a single operation [#4266](https://github.com/grafana/tempo/pull/4266) (@ndk) * [FEATURE] Discarded span logging `log_discarded_spans` [#3957](https://github.com/grafana/tempo/issues/3957) (@dastrobu) * [FEATURE] TraceQL support for instrumentation scope [#3967](https://github.com/grafana/tempo/pull/3967) (@ie-pham) * [FEATURE] Export cost attribution usage metrics from distributor [#4162](https://github.com/grafana/tempo/pull/4162) (@mdisibio) * [FEATURE] TraceQL metrics: avg_over_time [#4073](https://github.com/grafana/tempo/pull/4073) (@javiermolinar) +* [FEATURE] Limit tags and tag values search [#4320](https://github.com/grafana/tempo/pull/4320) (@javiermolinar) * [ENHANCEMENT] distributor: return trace id length when it is invalid [#4407](https://github.com/grafana/tempo/pull/4407) (@carles-grafana) * [ENHANCEMENT] Update to the latest dskit [#4341](https://github.com/grafana/tempo/pull/4341) (@dastrobu) * [ENHANCEMENT] Changed log level from INFO to DEBUG for the TempoDB Find operation using traceId to reduce excessive/unwanted logs in log search. [#4179](https://github.com/grafana/tempo/pull/4179) (@Aki0x137) diff --git a/docs/sources/tempo/api_docs/_index.md b/docs/sources/tempo/api_docs/_index.md index 346e85cf3d2..87a599712f2 100644 --- a/docs/sources/tempo/api_docs/_index.md +++ b/docs/sources/tempo/api_docs/_index.md @@ -360,6 +360,10 @@ Parameters: Optional. Along with `end`, defines a time range from which tags should be returned. - `end = (unix epoch seconds)` Optional. Along with `start`, defines a time range from which tags should be returned. Providing both `start` and `end` includes blocks for the specified time range only. +- `limit = (integer)` + Optional. Limits the maximum number of tags values. +- `maxStaleValues = (integer)` + Optional. Limits the search for tags names. If the number of stale (already known) values reaches or exceeds this limit, the search stops. i.e. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early. ### Search tags V2 @@ -385,6 +389,10 @@ Parameters: Optional. Along with `end` define a time range from which tags should be returned. - `end = (unix epoch seconds)` Optional. Along with `start` define a time range from which tags should be returned. Providing both `start` and `end` includes blocks for the specified time range only. +- `limit = (integer)` + Optional. Sets the maximum number of tags names allowed per scope. The query stops once this limit is reached for any scope. +- `maxStaleValues = (integer)` + Optional. Limits the search for tag values. If the number of stale (already known) values reaches or exceeds this limit, the search stops. #### Example @@ -515,6 +523,10 @@ Parameters: Optional. Along with `end`, defines a time range from which tags should be returned. - `end = (unix epoch seconds)` Optional. Along with `start`, defines a time range from which tags should be returned. Providing both `start` and `end` includes blocks for the specified time range only. +- `limit = (integer)` + Optional. Limits the maximum number of tags values. +- `maxStaleValues = (integer)` + Optional. Limits the search for tags values. If the number of stale (already known) values reaches or exceeds this limit, the search stops. i.e. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early. ### Search tag values V2 @@ -561,7 +573,17 @@ $ curl -G -s http://localhost:3200/api/v2/search/tag/.service.name/values | jq } } ``` -This endpoint can also receive `start` and `end` optional parameters. These parameters define the time range from which the tags are fetched +Parameters: +- `start = (unix epoch seconds)` + Optional. Along with `end`, defines a time range from which tags values should be returned. +- `end = (unix epoch seconds)` + Optional. Along with `start`, defines a time range from which tags values should be returned. Providing both `start` and `end` includes blocks for the specified time range only. +- `q = (traceql query)` + Optional. A TraceQL query to filter tag values by. Currently only works for a single spanset of `&&`ed conditions. For example: `{ span.foo = "bar" && resource.baz = "bat" ...}`. See also [Filtered tag values](#filtered-tag-values). +- `limit = (integer)` + Optional. Limits the maximum number of tags values +- `maxStaleValues = (integer)` + Optional. Limits the search for tags values. If the number of stale (already known) values reaches or exceeds this limit, the search stops. i.e. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early. #### Filtered tag values diff --git a/integration/e2e/multi_tenant_test.go b/integration/e2e/multi_tenant_test.go index 9b91c3b7c13..f63ca403526 100644 --- a/integration/e2e/multi_tenant_test.go +++ b/integration/e2e/multi_tenant_test.go @@ -207,9 +207,9 @@ func assertRequestCountMetric(t *testing.T, s *e2e.HTTPService, route string, re // getAttrsAndSpanNames returns trace attrs and span names func getAttrsAndSpanNames(trace *tempopb.Trace) traceStringsMap { - rAttrsKeys := collector.NewDistinctString(0) - rAttrsValues := collector.NewDistinctString(0) - spanNames := collector.NewDistinctString(0) + rAttrsKeys := collector.NewDistinctString(0, 0, 0) + rAttrsValues := collector.NewDistinctString(0, 0, 0) + spanNames := collector.NewDistinctString(0, 0, 0) for _, b := range trace.ResourceSpans { for _, ss := range b.ScopeSpans { diff --git a/modules/frontend/combiner/search_tag_values.go b/modules/frontend/combiner/search_tag_values.go index 253033dec33..23b68e8351b 100644 --- a/modules/frontend/combiner/search_tag_values.go +++ b/modules/frontend/combiner/search_tag_values.go @@ -12,9 +12,9 @@ var ( _ GRPCCombiner[*tempopb.SearchTagValuesV2Response] = (*genericCombiner[*tempopb.SearchTagValuesV2Response])(nil) ) -func NewSearchTagValues(limitBytes int) Combiner { +func NewSearchTagValues(maxDataBytes int, maxTagsValues uint32, staleValueThreshold uint32) Combiner { // Distinct collector with no limit - d := collector.NewDistinctStringWithDiff(limitBytes) + d := collector.NewDistinctStringWithDiff(maxDataBytes, maxTagsValues, staleValueThreshold) inspectedBytes := atomic.NewUint64(0) c := &genericCombiner[*tempopb.SearchTagValuesResponse]{ @@ -56,13 +56,13 @@ func NewSearchTagValues(limitBytes int) Combiner { return c } -func NewTypedSearchTagValues(limitBytes int) GRPCCombiner[*tempopb.SearchTagValuesResponse] { - return NewSearchTagValues(limitBytes).(GRPCCombiner[*tempopb.SearchTagValuesResponse]) +func NewTypedSearchTagValues(maxDataBytes int, maxTagsValues uint32, staleValueThreshold uint32) GRPCCombiner[*tempopb.SearchTagValuesResponse] { + return NewSearchTagValues(maxDataBytes, maxTagsValues, staleValueThreshold).(GRPCCombiner[*tempopb.SearchTagValuesResponse]) } -func NewSearchTagValuesV2(limitBytes int) Combiner { +func NewSearchTagValuesV2(maxDataBytes int, maxTagsValues uint32, staleValueThreshold uint32) Combiner { // Distinct collector with no limit and diff enabled - d := collector.NewDistinctValueWithDiff(limitBytes, func(tv tempopb.TagValue) int { return len(tv.Type) + len(tv.Value) }) + d := collector.NewDistinctValueWithDiff(maxDataBytes, maxTagsValues, staleValueThreshold, func(tv tempopb.TagValue) int { return len(tv.Type) + len(tv.Value) }) inspectedBytes := atomic.NewUint64(0) c := &genericCombiner[*tempopb.SearchTagValuesV2Response]{ @@ -113,6 +113,6 @@ func NewSearchTagValuesV2(limitBytes int) Combiner { return c } -func NewTypedSearchTagValuesV2(limitBytes int) GRPCCombiner[*tempopb.SearchTagValuesV2Response] { - return NewSearchTagValuesV2(limitBytes).(GRPCCombiner[*tempopb.SearchTagValuesV2Response]) +func NewTypedSearchTagValuesV2(maxDataBytes int, maxTagsValues uint32, staleValueThreshold uint32) GRPCCombiner[*tempopb.SearchTagValuesV2Response] { + return NewSearchTagValuesV2(maxDataBytes, maxTagsValues, staleValueThreshold).(GRPCCombiner[*tempopb.SearchTagValuesV2Response]) } diff --git a/modules/frontend/combiner/search_tags.go b/modules/frontend/combiner/search_tags.go index 224a5706b1a..8653536dc2d 100644 --- a/modules/frontend/combiner/search_tags.go +++ b/modules/frontend/combiner/search_tags.go @@ -12,8 +12,8 @@ var ( _ GRPCCombiner[*tempopb.SearchTagsV2Response] = (*genericCombiner[*tempopb.SearchTagsV2Response])(nil) ) -func NewSearchTags(limitBytes int) Combiner { - d := collector.NewDistinctStringWithDiff(limitBytes) +func NewSearchTags(maxDataBytes int, maxTagsPerScope uint32, staleValueThreshold uint32) Combiner { + d := collector.NewDistinctStringWithDiff(maxDataBytes, maxTagsPerScope, staleValueThreshold) inspectedBytes := atomic.NewUint64(0) c := &genericCombiner[*tempopb.SearchTagsResponse]{ @@ -56,13 +56,13 @@ func NewSearchTags(limitBytes int) Combiner { return c } -func NewTypedSearchTags(limitBytes int) GRPCCombiner[*tempopb.SearchTagsResponse] { - return NewSearchTags(limitBytes).(GRPCCombiner[*tempopb.SearchTagsResponse]) +func NewTypedSearchTags(maxDataBytes int, maxTagsPerScope uint32, staleValueThreshold uint32) GRPCCombiner[*tempopb.SearchTagsResponse] { + return NewSearchTags(maxDataBytes, maxTagsPerScope, staleValueThreshold).(GRPCCombiner[*tempopb.SearchTagsResponse]) } -func NewSearchTagsV2(limitBytes int) Combiner { +func NewSearchTagsV2(maxDataBytes int, maxTagsPerScope uint32, staleValueThreshold uint32) Combiner { // Distinct collector map to collect scopes and scope values - distinctValues := collector.NewScopedDistinctStringWithDiff(limitBytes) + distinctValues := collector.NewScopedDistinctStringWithDiff(maxDataBytes, maxTagsPerScope, staleValueThreshold) inspectedBytes := atomic.NewUint64(0) c := &genericCombiner[*tempopb.SearchTagsV2Response]{ @@ -121,6 +121,6 @@ func NewSearchTagsV2(limitBytes int) Combiner { return c } -func NewTypedSearchTagsV2(limitBytes int) GRPCCombiner[*tempopb.SearchTagsV2Response] { - return NewSearchTagsV2(limitBytes).(GRPCCombiner[*tempopb.SearchTagsV2Response]) +func NewTypedSearchTagsV2(maxDataBytes int, maxTagsPerScope uint32, staleValueThreshold uint32) GRPCCombiner[*tempopb.SearchTagsV2Response] { + return NewSearchTagsV2(maxDataBytes, maxTagsPerScope, staleValueThreshold).(GRPCCombiner[*tempopb.SearchTagsV2Response]) } diff --git a/modules/frontend/combiner/search_tags_test.go b/modules/frontend/combiner/search_tags_test.go index 3fed17bc8f3..cbac1dd118c 100644 --- a/modules/frontend/combiner/search_tags_test.go +++ b/modules/frontend/combiner/search_tags_test.go @@ -13,8 +13,10 @@ import ( func TestTagsCombiner(t *testing.T) { tests := []struct { name string - factory func(int) Combiner - limit int + factory func(int, uint32, uint32) Combiner + limitBytes int + maxTagsValues uint32 + maxCacheHits uint32 result1 proto.Message result2 proto.Message expectedResult proto.Message @@ -31,7 +33,7 @@ func TestTagsCombiner(t *testing.T) { expectedResult: &tempopb.SearchTagsResponse{TagNames: []string{"tag1", "tag2", "tag3"}, Metrics: &tempopb.MetadataMetrics{}}, actualResult: &tempopb.SearchTagsResponse{}, sort: func(m proto.Message) { sort.Strings(m.(*tempopb.SearchTagsResponse).TagNames) }, - limit: 100, + limitBytes: 100, }, { name: "SearchTagsV2", @@ -49,7 +51,7 @@ func TestTagsCombiner(t *testing.T) { return scopes[i].Name < scopes[j].Name }) }, - limit: 100, + limitBytes: 100, }, { name: "SearchTagValues", @@ -59,7 +61,7 @@ func TestTagsCombiner(t *testing.T) { expectedResult: &tempopb.SearchTagValuesResponse{TagValues: []string{"tag1", "tag2", "tag3"}, Metrics: &tempopb.MetadataMetrics{}}, actualResult: &tempopb.SearchTagValuesResponse{}, sort: func(m proto.Message) { sort.Strings(m.(*tempopb.SearchTagValuesResponse).TagValues) }, - limit: 100, + limitBytes: 100, }, { name: "SearchTagValuesV2", @@ -73,7 +75,7 @@ func TestTagsCombiner(t *testing.T) { return m.(*tempopb.SearchTagValuesV2Response).TagValues[i].Value < m.(*tempopb.SearchTagValuesV2Response).TagValues[j].Value }) }, - limit: 100, + limitBytes: 100, }, // limits { @@ -85,7 +87,7 @@ func TestTagsCombiner(t *testing.T) { actualResult: &tempopb.SearchTagsResponse{}, sort: func(m proto.Message) { sort.Strings(m.(*tempopb.SearchTagsResponse).TagNames) }, expectedShouldQuit: true, - limit: 5, + limitBytes: 5, }, { name: "SearchTagsV2 - limited", @@ -104,7 +106,7 @@ func TestTagsCombiner(t *testing.T) { }) }, expectedShouldQuit: true, - limit: 2, + limitBytes: 2, }, { name: "SearchTagValues - limited", @@ -115,7 +117,7 @@ func TestTagsCombiner(t *testing.T) { actualResult: &tempopb.SearchTagValuesResponse{}, sort: func(m proto.Message) { sort.Strings(m.(*tempopb.SearchTagValuesResponse).TagValues) }, expectedShouldQuit: true, - limit: 5, + limitBytes: 5, }, { name: "SearchTagValuesV2 - limited", @@ -130,7 +132,22 @@ func TestTagsCombiner(t *testing.T) { }) }, expectedShouldQuit: true, - limit: 10, + limitBytes: 10, + }, + { + name: "SearchTagValuesV2 - max values limited", + factory: NewSearchTagValuesV2, + result1: &tempopb.SearchTagValuesV2Response{TagValues: []*tempopb.TagValue{{Value: "v1", Type: "string"}}}, + result2: &tempopb.SearchTagValuesV2Response{TagValues: []*tempopb.TagValue{{Value: "v2", Type: "string"}, {Value: "v3", Type: "string"}}}, + expectedResult: &tempopb.SearchTagValuesV2Response{TagValues: []*tempopb.TagValue{{Value: "v1", Type: "string"}}, Metrics: &tempopb.MetadataMetrics{}}, + actualResult: &tempopb.SearchTagValuesV2Response{}, + sort: func(m proto.Message) { + sort.Slice(m.(*tempopb.SearchTagValuesV2Response).TagValues, func(i, j int) bool { + return m.(*tempopb.SearchTagValuesV2Response).TagValues[i].Value < m.(*tempopb.SearchTagValuesV2Response).TagValues[j].Value + }) + }, + expectedShouldQuit: true, + maxTagsValues: 1, }, // with metrics { @@ -141,7 +158,7 @@ func TestTagsCombiner(t *testing.T) { expectedResult: &tempopb.SearchTagsResponse{TagNames: []string{"tag1", "tag2", "tag3"}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 2}}, actualResult: &tempopb.SearchTagsResponse{}, sort: func(m proto.Message) { sort.Strings(m.(*tempopb.SearchTagsResponse).TagNames) }, - limit: 100, + limitBytes: 100, }, { name: "SearchTagsV2 - metrics", @@ -159,7 +176,7 @@ func TestTagsCombiner(t *testing.T) { return scopes[i].Name < scopes[j].Name }) }, - limit: 100, + limitBytes: 100, }, { name: "SearchTagValues - metrics", @@ -169,7 +186,7 @@ func TestTagsCombiner(t *testing.T) { expectedResult: &tempopb.SearchTagValuesResponse{TagValues: []string{"tag1", "tag2", "tag3"}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 2}}, actualResult: &tempopb.SearchTagValuesResponse{}, sort: func(m proto.Message) { sort.Strings(m.(*tempopb.SearchTagValuesResponse).TagValues) }, - limit: 100, + limitBytes: 100, }, { name: "SearchTagValuesV2 - metrics", @@ -183,12 +200,12 @@ func TestTagsCombiner(t *testing.T) { return m.(*tempopb.SearchTagValuesV2Response).TagValues[i].Value < m.(*tempopb.SearchTagValuesV2Response).TagValues[j].Value }) }, - limit: 100, + limitBytes: 100, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - combiner := tc.factory(tc.limit) + combiner := tc.factory(tc.limitBytes, tc.maxTagsValues, tc.maxCacheHits) err := combiner.AddResponse(toHTTPResponse(t, tc.result1, 200)) assert.NoError(t, err) @@ -228,7 +245,7 @@ func metrics(message proto.Message) *tempopb.MetadataMetrics { } func TestTagsGRPCCombiner(t *testing.T) { - c := NewTypedSearchTags(0) + c := NewTypedSearchTags(0, 0, 0) res1 := &tempopb.SearchTagsResponse{TagNames: []string{"tag1"}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} res2 := &tempopb.SearchTagsResponse{TagNames: []string{"tag1", "tag2"}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} diff1 := &tempopb.SearchTagsResponse{TagNames: []string{"tag1"}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} @@ -238,7 +255,7 @@ func TestTagsGRPCCombiner(t *testing.T) { } func TestTagsV2GRPCCombiner(t *testing.T) { - c := NewTypedSearchTagsV2(0) + c := NewTypedSearchTagsV2(0, 0, 0) res1 := &tempopb.SearchTagsV2Response{Scopes: []*tempopb.SearchTagsV2Scope{{Name: "scope1", Tags: []string{"tag1"}}}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} res2 := &tempopb.SearchTagsV2Response{Scopes: []*tempopb.SearchTagsV2Scope{{Name: "scope1", Tags: []string{"tag1", "tag2"}}, {Name: "scope2", Tags: []string{"tag3"}}}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} diff1 := &tempopb.SearchTagsV2Response{Scopes: []*tempopb.SearchTagsV2Scope{{Name: "scope1", Tags: []string{"tag1"}}}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} @@ -255,7 +272,7 @@ func TestTagsV2GRPCCombiner(t *testing.T) { } func TestTagValuesGRPCCombiner(t *testing.T) { - c := NewTypedSearchTagValues(0) + c := NewTypedSearchTagValues(0, 0, 0) res1 := &tempopb.SearchTagValuesResponse{TagValues: []string{"tag1"}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} res2 := &tempopb.SearchTagValuesResponse{TagValues: []string{"tag1", "tag2"}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} diff1 := &tempopb.SearchTagValuesResponse{TagValues: []string{"tag1"}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} @@ -265,7 +282,7 @@ func TestTagValuesGRPCCombiner(t *testing.T) { } func TestTagValuesV2GRPCCombiner(t *testing.T) { - c := NewTypedSearchTagValuesV2(0) + c := NewTypedSearchTagValuesV2(0, 0, 0) res1 := &tempopb.SearchTagValuesV2Response{TagValues: []*tempopb.TagValue{{Value: "v1", Type: "string"}}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} res2 := &tempopb.SearchTagValuesV2Response{TagValues: []*tempopb.TagValue{{Value: "v1", Type: "string"}, {Value: "v2", Type: "string"}}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} diff1 := &tempopb.SearchTagValuesV2Response{TagValues: []*tempopb.TagValue{{Value: "v1", Type: "string"}}, Metrics: &tempopb.MetadataMetrics{InspectedBytes: 1}} diff --git a/modules/frontend/search_sharder_test.go b/modules/frontend/search_sharder_test.go index 654eb389014..850f19b41a7 100644 --- a/modules/frontend/search_sharder_test.go +++ b/modules/frontend/search_sharder_test.go @@ -41,11 +41,11 @@ type mockReader struct { metas []*backend.BlockMeta } -func (m *mockReader) SearchTags(context.Context, *backend.BlockMeta, string, common.SearchOptions) (*tempopb.SearchTagsV2Response, error) { +func (m *mockReader) SearchTags(context.Context, *backend.BlockMeta, *tempopb.SearchTagsBlockRequest, common.SearchOptions) (*tempopb.SearchTagsV2Response, error) { return nil, nil } -func (m *mockReader) SearchTagValues(context.Context, *backend.BlockMeta, string, common.SearchOptions) (*tempopb.SearchTagValuesResponse, error) { +func (m *mockReader) SearchTagValues(context.Context, *backend.BlockMeta, *tempopb.SearchTagValuesBlockRequest, common.SearchOptions) (*tempopb.SearchTagValuesResponse, error) { return nil, nil } diff --git a/modules/frontend/tag_handlers.go b/modules/frontend/tag_handlers.go index 404009a48ca..e8d2d3f1431 100644 --- a/modules/frontend/tag_handlers.go +++ b/modules/frontend/tag_handlers.go @@ -47,8 +47,8 @@ func newTagsStreamingGRPCHandler(cfg Config, next pipeline.AsyncRoundTripper[com } var finalResponse *tempopb.SearchTagsResponse - comb := combiner.NewTypedSearchTags(o.MaxBytesPerTagValuesQuery(tenant)) - collector := pipeline.NewGRPCCollector[*tempopb.SearchTagsResponse](next, cfg.ResponseConsumers, comb, func(res *tempopb.SearchTagsResponse) error { + comb := combiner.NewTypedSearchTags(o.MaxBytesPerTagValuesQuery(tenant), req.MaxTagsPerScope, req.StaleValuesThreshold) + collector := pipeline.NewGRPCCollector(next, cfg.ResponseConsumers, comb, func(res *tempopb.SearchTagsResponse) error { finalResponse = res // to get the bytes processed for SLO calculations return srv.Send(res) }) @@ -80,8 +80,8 @@ func newTagsV2StreamingGRPCHandler(cfg Config, next pipeline.AsyncRoundTripper[c } var finalResponse *tempopb.SearchTagsV2Response - comb := combiner.NewTypedSearchTagsV2(o.MaxBytesPerTagValuesQuery(tenant)) - collector := pipeline.NewGRPCCollector[*tempopb.SearchTagsV2Response](next, cfg.ResponseConsumers, comb, func(res *tempopb.SearchTagsV2Response) error { + comb := combiner.NewTypedSearchTagsV2(o.MaxBytesPerTagValuesQuery(tenant), req.MaxTagsPerScope, req.StaleValuesThreshold) + collector := pipeline.NewGRPCCollector(next, cfg.ResponseConsumers, comb, func(res *tempopb.SearchTagsV2Response) error { finalResponse = res // to get the bytes processed for SLO calculations return srv.Send(res) }) @@ -117,8 +117,8 @@ func newTagValuesStreamingGRPCHandler(cfg Config, next pipeline.AsyncRoundTrippe } var finalResponse *tempopb.SearchTagValuesResponse - comb := combiner.NewTypedSearchTagValues(o.MaxBytesPerTagValuesQuery(tenant)) - collector := pipeline.NewGRPCCollector[*tempopb.SearchTagValuesResponse](next, cfg.ResponseConsumers, comb, func(res *tempopb.SearchTagValuesResponse) error { + comb := combiner.NewTypedSearchTagValues(o.MaxBytesPerTagValuesQuery(tenant), req.MaxTagValues, req.StaleValueThreshold) + collector := pipeline.NewGRPCCollector(next, cfg.ResponseConsumers, comb, func(res *tempopb.SearchTagValuesResponse) error { finalResponse = res // to get the bytes processed for SLO calculations return srv.Send(res) }) @@ -154,8 +154,8 @@ func newTagValuesV2StreamingGRPCHandler(cfg Config, next pipeline.AsyncRoundTrip } var finalResponse *tempopb.SearchTagValuesV2Response - comb := combiner.NewTypedSearchTagValuesV2(o.MaxBytesPerTagValuesQuery(tenant)) - collector := pipeline.NewGRPCCollector[*tempopb.SearchTagValuesV2Response](next, cfg.ResponseConsumers, comb, func(res *tempopb.SearchTagValuesV2Response) error { + comb := combiner.NewTypedSearchTagValuesV2(o.MaxBytesPerTagValuesQuery(tenant), req.MaxTagValues, req.StaleValueThreshold) + collector := pipeline.NewGRPCCollector(next, cfg.ResponseConsumers, comb, func(res *tempopb.SearchTagValuesV2Response) error { finalResponse = res // to get the bytes processed for SLO calculations return srv.Send(res) }) @@ -187,9 +187,9 @@ func newTagsHTTPHandler(cfg Config, next pipeline.AsyncRoundTripper[combiner.Pip return errResp, nil } - scope, _, rangeDur := parseParams(req) + scope, _, rangeDur, maxTagsPerScope, staleValueThreshold := parseParams(req) // build and use round tripper - comb := combiner.NewTypedSearchTags(o.MaxBytesPerTagValuesQuery(tenant)) + comb := combiner.NewTypedSearchTags(o.MaxBytesPerTagValuesQuery(tenant), maxTagsPerScope, staleValueThreshold) rt := pipeline.NewHTTPCollector(next, cfg.ResponseConsumers, comb) start := time.Now() logTagsRequest(logger, tenant, "SearchTags", scope, rangeDur) @@ -221,9 +221,9 @@ func newTagsV2HTTPHandler(cfg Config, next pipeline.AsyncRoundTripper[combiner.P return errResp, nil } - scope, _, rangeDur := parseParams(req) + scope, _, rangeDur, maxTagsPerScope, staleValueThreshold := parseParams(req) // build and use round tripper - comb := combiner.NewTypedSearchTagsV2(o.MaxBytesPerTagValuesQuery(tenant)) + comb := combiner.NewTypedSearchTagsV2(o.MaxBytesPerTagValuesQuery(tenant), maxTagsPerScope, staleValueThreshold) rt := pipeline.NewHTTPCollector(next, cfg.ResponseConsumers, comb) start := time.Now() logTagsRequest(logger, tenant, "SearchTagsV2", scope, rangeDur) @@ -255,11 +255,11 @@ func newTagValuesHTTPHandler(cfg Config, next pipeline.AsyncRoundTripper[combine return errResp, nil } - _, query, rangeDur := parseParams(req) + _, query, rangeDur, maxTagsValues, staleValueThreshold := parseParams(req) tagName := extractTagName(req.URL.Path, tagNameRegexV1) // build and use round tripper - comb := combiner.NewTypedSearchTagValues(o.MaxBytesPerTagValuesQuery(tenant)) + comb := combiner.NewTypedSearchTagValues(o.MaxBytesPerTagValuesQuery(tenant), maxTagsValues, staleValueThreshold) rt := pipeline.NewHTTPCollector(next, cfg.ResponseConsumers, comb) start := time.Now() logTagValuesRequest(logger, tenant, "SearchTagValues", tagName, query, rangeDur) @@ -291,11 +291,11 @@ func newTagValuesV2HTTPHandler(cfg Config, next pipeline.AsyncRoundTripper[combi return errResp, nil } - _, query, rangeDur := parseParams(req) + _, query, rangeDur, maxTagsValues, staleValueThreshold := parseParams(req) tagName := extractTagName(req.URL.Path, tagNameRegexV2) // build and use round tripper - comb := combiner.NewTypedSearchTagValuesV2(o.MaxBytesPerTagValuesQuery(tenant)) + comb := combiner.NewTypedSearchTagValuesV2(o.MaxBytesPerTagValuesQuery(tenant), maxTagsValues, staleValueThreshold) rt := pipeline.NewHTTPCollector(next, cfg.ResponseConsumers, comb) start := time.Now() logTagValuesRequest(logger, tenant, "SearchTagValuesV2", tagName, query, rangeDur) @@ -427,21 +427,26 @@ func logTagValuesResult(logger log.Logger, tenantID, handler, tagName, query str // parseParams parses optional 'start', 'end', 'scope', and 'q' params from a http.Request // returns scope, query and duration (end - start). returns "", and 0 if these params are invalid or absent -func parseParams(req *http.Request) (string, string, uint32) { +func parseParams(req *http.Request) (scope string, q string, duration uint32, limit uint32, staleValueThreshold uint32) { query := req.URL.Query() - - scope := query.Get("scope") - q := query.Get("q") + scope = query.Get("scope") + q = query.Get("q") // ignore errors, we default to 0 as params are not always present. start, _ := strconv.ParseInt(query.Get("start"), 10, 64) end, _ := strconv.ParseInt(query.Get("end"), 10, 64) - - var duration int64 + maxItems, err := strconv.ParseUint(query.Get("limit"), 10, 32) + if err != nil { + maxItems = 0 + } + maxStaleValues, err := strconv.ParseUint(query.Get("maxStaleValues"), 10, 32) + if err != nil { + maxStaleValues = 0 + } // duration only makes sense if start and end are present and end is greater than start if start > 0 && end > 0 && end > start { - duration = end - start + duration = uint32(end - start) } - return scope, q, uint32(duration) + return scope, q, duration, uint32(maxItems), uint32(maxStaleValues) } // extractTagName extracts the tagName based on the provided regex pattern diff --git a/modules/frontend/tag_handlers_test.go b/modules/frontend/tag_handlers_test.go index 77915e55b72..658f5e91b2b 100644 --- a/modules/frontend/tag_handlers_test.go +++ b/modules/frontend/tag_handlers_test.go @@ -538,88 +538,148 @@ func TestSearchTagsV2AccessesCache(t *testing.T) { func TestParseParams(t *testing.T) { tests := []struct { - name string - queryParams map[string]string - expectedScope string - expectedQ string - expectedDuration uint32 + name string + queryParams map[string]string + expectedScope string + expectedQ string + expectedDuration uint32 + expectedLimit uint32 + expectedValueThreshold uint32 }{ { - name: "all params present", - queryParams: map[string]string{"start": "1723667082", "end": "1723839882", "scope": "resource", "q": "some_query"}, - expectedScope: "resource", - expectedQ: "some_query", - expectedDuration: 172800, + name: "all params present", + queryParams: map[string]string{"start": "1723667082", "end": "1723839882", "scope": "resource", "q": "some_query"}, + expectedScope: "resource", + expectedQ: "some_query", + expectedDuration: 172800, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "missing start", - queryParams: map[string]string{"end": "1723839882", "scope": "resource"}, - expectedScope: "resource", - expectedQ: "", - expectedDuration: 0, + name: "missing start", + queryParams: map[string]string{"end": "1723839882", "scope": "resource"}, + expectedScope: "resource", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "missing end", - queryParams: map[string]string{"start": "1723667082", "scope": "resource"}, - expectedScope: "resource", - expectedQ: "", - expectedDuration: 0, + name: "missing end", + queryParams: map[string]string{"start": "1723667082", "scope": "resource"}, + expectedScope: "resource", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "missing scope", - queryParams: map[string]string{"start": "1723667082", "end": "1723839882"}, - expectedScope: "", - expectedQ: "", - expectedDuration: 172800, + name: "missing scope", + queryParams: map[string]string{"start": "1723667082", "end": "1723839882"}, + expectedScope: "", + expectedQ: "", + expectedDuration: 172800, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "missing q", - queryParams: map[string]string{"start": "1723667082", "end": "1723839882", "scope": "resource"}, - expectedScope: "resource", - expectedQ: "", - expectedDuration: 172800, + name: "missing q", + queryParams: map[string]string{"start": "1723667082", "end": "1723839882", "scope": "resource"}, + expectedScope: "resource", + expectedQ: "", + expectedDuration: 172800, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "invalid start", - queryParams: map[string]string{"start": "invalid", "end": "1723839882", "scope": "resource"}, - expectedScope: "resource", - expectedQ: "", - expectedDuration: 0, + name: "invalid start", + queryParams: map[string]string{"start": "invalid", "end": "1723839882", "scope": "resource"}, + expectedScope: "resource", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "invalid end", - queryParams: map[string]string{"start": "1723667082", "end": "invalid", "scope": "resource"}, - expectedScope: "resource", - expectedQ: "", - expectedDuration: 0, + name: "invalid end", + queryParams: map[string]string{"start": "1723667082", "end": "invalid", "scope": "resource"}, + expectedScope: "resource", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "no params", - queryParams: map[string]string{}, - expectedScope: "", - expectedQ: "", - expectedDuration: 0, + name: "no params", + queryParams: map[string]string{}, + expectedScope: "", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "negative start and end", - queryParams: map[string]string{"start": "-1000", "end": "-2000", "scope": "negative_case"}, - expectedScope: "negative_case", - expectedQ: "", - expectedDuration: 0, + name: "negative start and end", + queryParams: map[string]string{"start": "-1000", "end": "-2000", "scope": "negative_case"}, + expectedScope: "negative_case", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "end less than start", - queryParams: map[string]string{"start": "1723839882", "end": "1723667082", "scope": "resource"}, - expectedScope: "resource", - expectedQ: "", - expectedDuration: 0, + name: "end less than start", + queryParams: map[string]string{"start": "1723839882", "end": "1723667082", "scope": "resource"}, + expectedScope: "resource", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, }, { - name: "start and end are the same", - queryParams: map[string]string{"start": "1723839882", "end": "1723839882", "scope": "zero_duration"}, - expectedScope: "zero_duration", - expectedQ: "", - expectedDuration: 0, + name: "start and end are the same", + queryParams: map[string]string{"start": "1723839882", "end": "1723839882", "scope": "zero_duration"}, + expectedScope: "zero_duration", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, + }, + { + name: "invalid limit", + queryParams: map[string]string{"limit": "-1000"}, + expectedScope: "", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, + }, + { + name: "invalid too large limit", + queryParams: map[string]string{"limit": "1000000000000000000"}, + expectedScope: "", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 0, + }, + { + name: "valid limit", + queryParams: map[string]string{"limit": "100"}, + expectedScope: "", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 100, + expectedValueThreshold: 0, + }, + { + name: "valid threshold", + queryParams: map[string]string{"maxStaleValues": "100"}, + expectedScope: "", + expectedQ: "", + expectedDuration: 0, + expectedLimit: 0, + expectedValueThreshold: 100, }, } @@ -633,11 +693,13 @@ func TestParseParams(t *testing.T) { u.RawQuery = query.Encode() req := &http.Request{URL: u} - scope, q, duration := parseParams(req) + scope, q, duration, limit, staleValueThreshold := parseParams(req) require.Equal(t, tt.expectedScope, scope) require.Equal(t, tt.expectedQ, q) require.Equal(t, tt.expectedDuration, duration) + require.Equal(t, tt.expectedLimit, limit) + require.Equal(t, tt.expectedValueThreshold, staleValueThreshold) }) } } diff --git a/modules/ingester/ingester_search.go b/modules/ingester/ingester_search.go index d87af8b429b..6398b93e48f 100644 --- a/modules/ingester/ingester_search.go +++ b/modules/ingester/ingester_search.go @@ -103,7 +103,7 @@ func (i *Ingester) SearchTagValues(ctx context.Context, req *tempopb.SearchTagVa return &tempopb.SearchTagValuesResponse{}, nil } - res, err = inst.SearchTagValues(ctx, req.TagName) + res, err = inst.SearchTagValues(ctx, req.TagName, req.MaxTagValues, req.StaleValueThreshold) if err != nil { return nil, err } diff --git a/modules/ingester/instance_search.go b/modules/ingester/instance_search.go index f556ad5b61b..ee6cf964bcc 100644 --- a/modules/ingester/instance_search.go +++ b/modules/ingester/instance_search.go @@ -182,7 +182,7 @@ func (i *instance) SearchTags(ctx context.Context, scope string) (*tempopb.Searc return nil, err } - distinctValues := collector.NewDistinctString(0) // search tags v2 enforces the limit + distinctValues := collector.NewDistinctString(0, 0, 0) // search tags v2 enforces the limit // flatten v2 response for _, s := range v2Response.Scopes { @@ -232,8 +232,8 @@ func (i *instance) SearchTagsV2(ctx context.Context, req *tempopb.SearchTagsRequ return nil, fmt.Errorf("unknown scope: %s", scope) } - limit := i.limiter.limits.MaxBytesPerTagValuesQuery(userID) - distinctValues := collector.NewScopedDistinctString(limit) + maxBytestPerTags := i.limiter.limits.MaxBytesPerTagValuesQuery(userID) + distinctValues := collector.NewScopedDistinctString(maxBytestPerTags, req.MaxTagsPerScope, req.StaleValuesThreshold) mc := collector.NewMetricsCollector() engine := traceql.NewEngine() @@ -296,7 +296,7 @@ func (i *instance) SearchTagsV2(ctx context.Context, req *tempopb.SearchTagsRequ } if distinctValues.Exceeded() { - level.Warn(log.Logger).Log("msg", "size of tags in instance exceeded limit, reduce cardinality or size of tags", "tenant", userID, "limit", limit) + level.Warn(log.Logger).Log("msg", "Search of tags exceeded limit, reduce cardinality or size of tags", "orgID", userID, "stopReason", distinctValues.StopReason()) } collected := distinctValues.Strings() @@ -324,14 +324,14 @@ func (i *instance) SearchTagsV2(ctx context.Context, req *tempopb.SearchTagsRequ return resp, nil } -func (i *instance) SearchTagValues(ctx context.Context, tagName string) (*tempopb.SearchTagValuesResponse, error) { +func (i *instance) SearchTagValues(ctx context.Context, tagName string, limit uint32, staleValueThreshold uint32) (*tempopb.SearchTagValuesResponse, error) { userID, err := user.ExtractOrgID(ctx) if err != nil { return nil, err } - limit := i.limiter.limits.MaxBytesPerTagValuesQuery(userID) - distinctValues := collector.NewDistinctString(limit) + maxBytesPerTagValues := i.limiter.limits.MaxBytesPerTagValuesQuery(userID) + distinctValues := collector.NewDistinctString(maxBytesPerTagValues, limit, staleValueThreshold) mc := collector.NewMetricsCollector() var inspectedBlocks, maxBlocks int @@ -382,7 +382,7 @@ func (i *instance) SearchTagValues(ctx context.Context, tagName string) (*tempop } if distinctValues.Exceeded() { - level.Warn(log.Logger).Log("msg", "size of tag values in instance exceeded limit, reduce cardinality or size of tags", "tag", tagName, "tenant", userID, "limit", limit, "size", distinctValues.Size()) + level.Warn(log.Logger).Log("msg", "Search of tags exceeded limit, reduce cardinality or size of tags", "tag", tagName, "orgID", userID, "stopReason", distinctValues.StopReason()) } return &tempopb.SearchTagValuesResponse{ @@ -401,7 +401,7 @@ func (i *instance) SearchTagValuesV2(ctx context.Context, req *tempopb.SearchTag defer span.End() limit := i.limiter.limits.MaxBytesPerTagValuesQuery(userID) - valueCollector := collector.NewDistinctValue[tempopb.TagValue](limit, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + valueCollector := collector.NewDistinctValue(limit, req.MaxTagValues, req.StaleValueThreshold, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) mc := collector.NewMetricsCollector() // to collect bytesRead metric engine := traceql.NewEngine() @@ -512,7 +512,7 @@ func (i *instance) SearchTagValuesV2(ctx context.Context, req *tempopb.SearchTag // cache miss, search the block. We will cache the results if we find any. span.SetAttributes(attribute.Bool("cached", false)) // using local collector to collect values from the block and cache them. - localCol := collector.NewDistinctValue[tempopb.TagValue](limit, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + localCol := collector.NewDistinctValue[tempopb.TagValue](limit, req.MaxTagValues, req.StaleValueThreshold, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) localErr := performSearch(ctx, b, localCol) if localErr != nil { return localErr diff --git a/modules/ingester/instance_search_test.go b/modules/ingester/instance_search_test.go index 98636601446..cb4bada0a12 100644 --- a/modules/ingester/instance_search_test.go +++ b/modules/ingester/instance_search_test.go @@ -292,7 +292,7 @@ func testSearchTagsAndValues(t *testing.T, ctx context.Context, i *instance, tag checkSearchTags("event", true) checkSearchTags("link", true) - srv, err := i.SearchTagValues(ctx, tagName) + srv, err := i.SearchTagValues(ctx, tagName, 0, 0) require.NoError(t, err) require.Greater(t, srv.Metrics.InspectedBytes, uint64(100)) // we scanned at-least 100 bytes @@ -454,7 +454,7 @@ func TestInstanceSearchMaxBytesPerTagValuesQueryReturnsPartial(t *testing.T) { limits, err := overrides.NewOverrides(overrides.Config{ Defaults: overrides.Overrides{ Read: overrides.ReadOverrides{ - MaxBytesPerTagValuesQuery: 10, + MaxBytesPerTagValuesQuery: 12, }, }, }, nil, prometheus.DefaultRegisterer) @@ -474,9 +474,9 @@ func TestInstanceSearchMaxBytesPerTagValuesQueryReturnsPartial(t *testing.T) { _, _, _, _ = writeTracesForSearch(t, i, "", tagKey, tagValue, true, false) userCtx := user.InjectOrgID(context.Background(), "fake") - resp, err := i.SearchTagValues(userCtx, tagKey) + resp, err := i.SearchTagValues(userCtx, tagKey, 0, 0) require.NoError(t, err) - require.Equal(t, 2, len(resp.TagValues)) // Only two values of the form "bar123" fit in the 10 byte limit above. + require.Equal(t, 2, len(resp.TagValues)) // Only two values of the form "bar123" fit in the 12 byte limit above. } // TestInstanceSearchMaxBytesPerTagValuesQueryReturnsPartial confirms that SearchTagValues returns @@ -514,7 +514,7 @@ func TestInstanceSearchMaxBlocksPerTagValuesQueryReturnsPartial(t *testing.T) { userCtx := user.InjectOrgID(context.Background(), "fake") - respV1, err := i.SearchTagValues(userCtx, tagKey) + respV1, err := i.SearchTagValues(userCtx, tagKey, 0, 0) require.NoError(t, err) assert.Equal(t, 100, len(respV1.TagValues)) @@ -528,7 +528,7 @@ func TestInstanceSearchMaxBlocksPerTagValuesQueryReturnsPartial(t *testing.T) { i.limiter = NewLimiter(limits, &ringCountMock{count: 1}, 1) - respV1, err = i.SearchTagValues(userCtx, tagKey) + respV1, err = i.SearchTagValues(userCtx, tagKey, 0, 0) require.NoError(t, err) assert.Equal(t, 200, len(respV1.TagValues)) @@ -722,7 +722,7 @@ func TestInstanceSearchDoesNotRace(t *testing.T) { go concurrent(func() { // SearchTagValues queries now require userID in ctx ctx := user.InjectOrgID(context.Background(), "test") - _, err := i.SearchTagValues(ctx, tagKey) + _, err := i.SearchTagValues(ctx, tagKey, 0, 0) require.NoError(t, err, "error getting search tag values") }) diff --git a/modules/querier/querier.go b/modules/querier/querier.go index 513f1fd0973..8f54f5e8b82 100644 --- a/modules/querier/querier.go +++ b/modules/querier/querier.go @@ -494,7 +494,7 @@ func (q *Querier) SearchTagsBlocks(ctx context.Context, req *tempopb.SearchTagsB return nil, err } - distinctValues := collector.NewDistinctString(0) + distinctValues := collector.NewDistinctString(0, 0, 0) // flatten v2 response for _, s := range v2Response.Scopes { @@ -535,8 +535,8 @@ func (q *Querier) SearchTags(ctx context.Context, req *tempopb.SearchTagsRequest return nil, fmt.Errorf("error extracting org id in Querier.SearchTags: %w", err) } - limit := q.limits.MaxBytesPerTagValuesQuery(userID) - distinctValues := collector.NewDistinctString(limit) + maxDataSize := q.limits.MaxBytesPerTagValuesQuery(userID) + distinctValues := collector.NewDistinctString(maxDataSize, req.MaxTagsPerScope, req.StaleValuesThreshold) mc := collector.NewMetricsCollector() forEach := func(ctx context.Context, client tempopb.QuerierClient) error { @@ -563,7 +563,7 @@ func (q *Querier) SearchTags(ctx context.Context, req *tempopb.SearchTagsRequest } if distinctValues.Exceeded() { - level.Warn(log.Logger).Log("msg", "size of tags in instance exceeded limit, reduce cardinality or size of tags", "userID", userID, "limit", limit, "size", distinctValues.Size()) + level.Warn(log.Logger).Log("msg", "size of tags in instance exceeded limit, reduce cardinality or size of tags", "userID", userID, "maxDataSize", maxDataSize, "size", distinctValues.Size()) } return &tempopb.SearchTagsResponse{ @@ -573,13 +573,13 @@ func (q *Querier) SearchTags(ctx context.Context, req *tempopb.SearchTagsRequest } func (q *Querier) SearchTagsV2(ctx context.Context, req *tempopb.SearchTagsRequest) (*tempopb.SearchTagsV2Response, error) { - userID, err := user.ExtractOrgID(ctx) + orgID, err := user.ExtractOrgID(ctx) if err != nil { return nil, fmt.Errorf("error extracting org id in Querier.SearchTags: %w", err) } - limit := q.limits.MaxBytesPerTagValuesQuery(userID) - distinctValues := collector.NewScopedDistinctString(limit) + maxBytesPerTag := q.limits.MaxBytesPerTagValuesQuery(orgID) + distinctValues := collector.NewScopedDistinctString(maxBytesPerTag, req.MaxTagsPerScope, req.StaleValuesThreshold) mc := collector.NewMetricsCollector() // Get results from all ingesters @@ -603,13 +603,13 @@ func (q *Querier) SearchTagsV2(ctx context.Context, req *tempopb.SearchTagsReque return nil } - err = q.forIngesterRings(ctx, userID, nil, forEach) + err = q.forIngesterRings(ctx, orgID, nil, forEach) if err != nil { return nil, fmt.Errorf("error querying ingesters in Querier.SearchTags: %w", err) } if distinctValues.Exceeded() { - level.Warn(log.Logger).Log("msg", "size of tags in instance exceeded limit, reduce cardinality or size of tags", "userID", userID, "limit", limit) + level.Warn(log.Logger).Log("msg", "Search tags exceeded limit, reduce cardinality or size of tags", "orgID", orgID, "stopReason", distinctValues.StopReason()) } collected := distinctValues.Strings() @@ -633,8 +633,8 @@ func (q *Querier) SearchTagValues(ctx context.Context, req *tempopb.SearchTagVal return nil, fmt.Errorf("error extracting org id in Querier.SearchTagValues: %w", err) } - limit := q.limits.MaxBytesPerTagValuesQuery(userID) - distinctValues := collector.NewDistinctString(limit) + maxDataSize := q.limits.MaxBytesPerTagValuesQuery(userID) + distinctValues := collector.NewDistinctString(maxDataSize, req.MaxTagValues, req.StaleValueThreshold) mc := collector.NewMetricsCollector() // Virtual tags values. Get these first. @@ -668,7 +668,7 @@ func (q *Querier) SearchTagValues(ctx context.Context, req *tempopb.SearchTagVal } if distinctValues.Exceeded() { - level.Warn(log.Logger).Log("msg", "size of tag values in instance exceeded limit, reduce cardinality or size of tags", "tag", req.TagName, "userID", userID, "limit", limit, "size", distinctValues.Size()) + level.Warn(log.Logger).Log("msg", "Search of tag values exceeded limit, reduce cardinality or size of tags", "tag", req.TagName, "orgID", userID, "stopReason", distinctValues.StopReason()) } return &tempopb.SearchTagValuesResponse{ @@ -683,8 +683,8 @@ func (q *Querier) SearchTagValuesV2(ctx context.Context, req *tempopb.SearchTagV return nil, fmt.Errorf("error extracting org id in Querier.SearchTagValues: %w", err) } - limit := q.limits.MaxBytesPerTagValuesQuery(userID) - distinctValues := collector.NewDistinctValue(limit, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + maxDataSize := q.limits.MaxBytesPerTagValuesQuery(userID) + distinctValues := collector.NewDistinctValue(maxDataSize, req.MaxTagValues, req.StaleValueThreshold, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) mc := collector.NewMetricsCollector() // Virtual tags values. Get these first. @@ -726,7 +726,7 @@ func (q *Querier) SearchTagValuesV2(ctx context.Context, req *tempopb.SearchTagV } if distinctValues.Exceeded() { - _ = level.Warn(log.Logger).Log("msg", "size of tag values exceeded limit, reduce cardinality or size of tags", "tag", req.TagName, "userID", userID, "limit", limit, "size", distinctValues.Size()) + _ = level.Warn(log.Logger).Log("msg", "Search of tag values exceeded limit, reduce cardinality or size of tags", "tag", req.TagName, "orgID", userID, "stopReason", distinctValues.StopReason()) } return valuesToV2Response(distinctValues, mc.TotalValue()), nil @@ -965,7 +965,7 @@ func (q *Querier) internalTagsSearchBlockV2(ctx context.Context, req *tempopb.Se query := traceql.ExtractMatchers(req.SearchReq.Query) if traceql.IsEmptyQuery(query) { - resp, err := q.store.SearchTags(ctx, meta, req.SearchReq.Scope, opts) + resp, err := q.store.SearchTags(ctx, meta, req, opts) if err != nil { return nil, err } @@ -981,7 +981,7 @@ func (q *Querier) internalTagsSearchBlockV2(ctx context.Context, req *tempopb.Se return resp, nil } - valueCollector := collector.NewScopedDistinctString(q.limits.MaxBytesPerTagValuesQuery(tenantID)) + valueCollector := collector.NewScopedDistinctString(q.limits.MaxBytesPerTagValuesQuery(tenantID), req.MaxTagsPerScope, req.StaleValueThreshold) mc := collector.NewMetricsCollector() fetcher := traceql.NewTagNamesFetcherWrapper(func(ctx context.Context, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback) error { @@ -1000,6 +1000,10 @@ func (q *Querier) internalTagsSearchBlockV2(ctx context.Context, req *tempopb.Se return nil, err } + if valueCollector.Exceeded() { + level.Warn(log.Logger).Log("msg", "Search tags exceeded limit, reduce cardinality or size of tags", "orgID", tenantID, "stopReason", valueCollector.StopReason()) + } + scopedVals := valueCollector.Strings() resp := &tempopb.SearchTagsV2Response{ Scopes: make([]*tempopb.SearchTagsV2Scope, 0, len(scopedVals)), @@ -1053,7 +1057,7 @@ func (q *Querier) internalTagValuesSearchBlock(ctx context.Context, req *tempopb opts.StartPage = int(req.StartPage) opts.TotalPages = int(req.PagesToSearch) - resp, err := q.store.SearchTagValues(ctx, meta, req.SearchReq.TagName, opts) + resp, err := q.store.SearchTagValues(ctx, meta, req, opts) if err != nil { return &tempopb.SearchTagValuesResponse{}, err } @@ -1109,7 +1113,10 @@ func (q *Querier) internalTagValuesSearchBlockV2(ctx context.Context, req *tempo return nil, err } - valueCollector := collector.NewDistinctValue(q.limits.MaxBytesPerTagValuesQuery(tenantID), func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + valueCollector := collector.NewDistinctValue(q.limits.MaxBytesPerTagValuesQuery(tenantID), + req.SearchReq.MaxTagValues, req.SearchReq.StaleValueThreshold, + func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + mc := collector.NewMetricsCollector() fetcher := traceql.NewTagValuesFetcherWrapper(func(ctx context.Context, req traceql.FetchTagValuesRequest, cb traceql.FetchTagValuesCallback) error { @@ -1121,6 +1128,10 @@ func (q *Querier) internalTagValuesSearchBlockV2(ctx context.Context, req *tempo return nil, err } + if valueCollector.Exceeded() { + level.Warn(log.Logger).Log("msg", "Search tags exceeded limit, reduce cardinality or size of tags", "orgID", tenantID, "stopReason", valueCollector.StopReason()) + } + return valuesToV2Response(valueCollector, mc.TotalValue()), nil } diff --git a/pkg/collector/distinct_string_collector.go b/pkg/collector/distinct_string_collector.go index ca881703e26..120ad4f6fe6 100644 --- a/pkg/collector/distinct_string_collector.go +++ b/pkg/collector/distinct_string_collector.go @@ -1,39 +1,51 @@ package collector import ( + "fmt" "sort" "strings" "sync" ) type DistinctString struct { - values map[string]struct{} - new map[string]struct{} - maxLen int - currLen int - diffEnabled bool - limExceeded bool - mtx sync.Mutex + values map[string]struct{} + new map[string]struct{} + maxDataSize int + currDataSize int + currentValuesLen uint32 + maxValues uint32 + maxCacheHits uint32 + currentCacheHits uint32 + diffEnabled bool + limExceeded bool + mtx sync.Mutex + stopReason string } -// NewDistinctString with the given maximum data size. This is calculated -// as the total length of the recorded strings. For ease of use, maximum=0 -// is interpreted as unlimited. -func NewDistinctString(maxDataSize int) *DistinctString { +// NewDistinctString with the given maximum data size and max items. +// MaxDataSize is calculated as the total length of the recorded strings. +// For ease of use, maxDataSize=0 and maxItems=0 are interpreted as unlimited. +func NewDistinctString(maxDataSize int, maxValues uint32, staleValueThreshold uint32) *DistinctString { return &DistinctString{ - values: make(map[string]struct{}), - maxLen: maxDataSize, - diffEnabled: false, // disable diff to make it faster + values: make(map[string]struct{}), + maxDataSize: maxDataSize, + diffEnabled: false, // disable diff to make it faster + maxValues: maxValues, + maxCacheHits: staleValueThreshold, } } // NewDistinctStringWithDiff is like NewDistinctString but with diff support enabled. -func NewDistinctStringWithDiff(maxDataSize int) *DistinctString { +// MaxDataSize is calculated as the total length of the recorded strings. +// For ease of use, maxDataSize=0 and maxItems=0 are interpreted as unlimited. +func NewDistinctStringWithDiff(maxDataSize int, maxValues uint32, staleValueThreshold uint32) *DistinctString { return &DistinctString{ - values: make(map[string]struct{}), - new: make(map[string]struct{}), - maxLen: maxDataSize, - diffEnabled: true, + values: make(map[string]struct{}), + new: make(map[string]struct{}), + maxDataSize: maxDataSize, + diffEnabled: true, + maxValues: maxValues, + maxCacheHits: staleValueThreshold, } } @@ -47,20 +59,33 @@ func (d *DistinctString) Collect(s string) (added bool) { if d.limExceeded { return false } + valueLen := len(s) - if _, ok := d.values[s]; ok { - // Already present + if d.maxDataSize > 0 && d.currDataSize+valueLen >= d.maxDataSize { + d.stopReason = fmt.Sprintf("Max data exceeded: dataSize %d, maxDataSize %d", d.currDataSize, d.maxDataSize) + d.limExceeded = true return false } - valueLen := len(s) - // Can it fit? - if d.maxLen > 0 && d.currLen+valueLen > d.maxLen { - // No, it can't fit + if d.maxValues > 0 && d.currentValuesLen >= d.maxValues { + d.stopReason = fmt.Sprintf("Max values exceeded: values %d, maxValues %d", d.currentValuesLen, d.maxValues) + d.limExceeded = true + return false + } + + if d.maxCacheHits > 0 && d.currentCacheHits >= d.maxCacheHits { + d.stopReason = fmt.Sprintf("Max stale values exceeded: cacheHits %d, maxValues %d", d.currentValuesLen, d.maxCacheHits) d.limExceeded = true return false } + if _, ok := d.values[s]; ok { + // Already present + d.currentCacheHits++ + return false + } + d.currentCacheHits = 0 // CacheHits reset to 0 when a new value is found + // Clone instead of referencing original s = strings.Clone(s) @@ -68,7 +93,8 @@ func (d *DistinctString) Collect(s string) (added bool) { d.new[s] = struct{}{} } d.values[s] = struct{}{} - d.currLen += valueLen + d.currDataSize += valueLen + d.currentValuesLen++ return true } @@ -96,12 +122,16 @@ func (d *DistinctString) Exceeded() bool { return d.limExceeded } +func (d *DistinctString) StopReason() string { + return d.stopReason +} + // Size is the total size of all distinct strings encountered. func (d *DistinctString) Size() int { d.mtx.Lock() defer d.mtx.Unlock() - return d.currLen + return d.currDataSize } // Diff returns all new strings collected since the last time diff was called diff --git a/pkg/collector/distinct_string_collector_test.go b/pkg/collector/distinct_string_collector_test.go index ec778e3012c..3e119d3a74c 100644 --- a/pkg/collector/distinct_string_collector_test.go +++ b/pkg/collector/distinct_string_collector_test.go @@ -10,12 +10,12 @@ import ( ) func TestDistinctStringCollector(t *testing.T) { - d := NewDistinctString(10) + d := NewDistinctString(12, 0, 0) - d.Collect("123") - d.Collect("4567") - d.Collect("890") - d.Collect("11") + require.True(t, d.Collect("123")) + require.True(t, d.Collect("4567")) + require.True(t, d.Collect("890")) + require.False(t, d.Collect("11")) require.True(t, d.Exceeded()) stringsSlicesEqual(t, []string{"123", "4567", "890"}, d.Strings()) @@ -26,17 +26,56 @@ func TestDistinctStringCollector(t *testing.T) { require.Error(t, err, errDiffNotEnabled) } +func TestDistinctStringCollectorWithMaxItemsLimit(t *testing.T) { + d := NewDistinctString(0, 3, 0) + + require.True(t, d.Collect("123")) + require.True(t, d.Collect("4567")) + require.True(t, d.Collect("890")) + require.False(t, d.Collect("11")) + + require.True(t, d.Exceeded()) + stringsSlicesEqual(t, []string{"123", "4567", "890"}, d.Strings()) + + // diff fails when diff is not enabled + res, err := d.Diff() + require.Nil(t, res) + require.Error(t, err, errDiffNotEnabled) +} + +func TestDistinctStringCollectorWitCacheHitsLimit(t *testing.T) { + d := NewDistinctString(0, 0, 3) + + require.True(t, d.Collect("123")) + require.True(t, d.Collect("4567")) + require.True(t, d.Collect("890")) + require.False(t, d.Collect("890")) + require.True(t, d.Collect("11")) // The counter resets with every new value + require.False(t, d.Collect("890")) + require.False(t, d.Collect("890")) + require.False(t, d.Collect("890")) + require.False(t, d.Collect("12")) + + require.True(t, d.Exceeded()) + stringsSlicesEqual(t, []string{"123", "4567", "890", "11"}, d.Strings()) + + // diff fails when diff is not enabled + res, err := d.Diff() + require.Nil(t, res) + require.Error(t, err, errDiffNotEnabled) +} + func TestDistinctStringCollectorDiff(t *testing.T) { - d := NewDistinctStringWithDiff(0) + d := NewDistinctStringWithDiff(0, 0, 0) - d.Collect("123") - d.Collect("4567") + require.True(t, d.Collect("123")) + require.True(t, d.Collect("4567")) stringsSlicesEqual(t, []string{"123", "4567"}, readDistinctStringDiff(t, d)) stringsSlicesEqual(t, []string{}, readDistinctStringDiff(t, d)) - d.Collect("123") - d.Collect("890") + require.False(t, d.Collect("123")) + require.True(t, d.Collect("890")) stringsSlicesEqual(t, []string{"890"}, readDistinctStringDiff(t, d)) stringsSlicesEqual(t, []string{}, readDistinctStringDiff(t, d)) @@ -49,7 +88,7 @@ func readDistinctStringDiff(t *testing.T, d *DistinctString) []string { } func TestDistinctStringCollectorIsSafe(t *testing.T) { - d := NewDistinctString(0) // no limit + d := NewDistinctString(0, 0, 0) // no limit var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) @@ -90,7 +129,7 @@ func BenchmarkDistinctStringCollect(b *testing.B) { for _, lim := range limits { b.Run("uniques_limit:"+strconv.Itoa(lim), func(b *testing.B) { for n := 0; n < b.N; n++ { - distinctStrings := NewDistinctString(lim) + distinctStrings := NewDistinctString(lim, 0, 0) for _, values := range ingesterStrings { for _, v := range values { if distinctStrings.Collect(v) { @@ -103,7 +142,7 @@ func BenchmarkDistinctStringCollect(b *testing.B) { b.Run("duplicates_limit:"+strconv.Itoa(lim), func(b *testing.B) { for n := 0; n < b.N; n++ { - distinctStrings := NewDistinctString(lim) + distinctStrings := NewDistinctString(lim, 0, 0) for i := 0; i < numIngesters; i++ { for j := 0; j < numTagValuesPerIngester; j++ { // collect first item to simulate duplicates diff --git a/pkg/collector/distinct_value_collector.go b/pkg/collector/distinct_value_collector.go index a8359667639..2df7561f58d 100644 --- a/pkg/collector/distinct_value_collector.go +++ b/pkg/collector/distinct_value_collector.go @@ -2,43 +2,54 @@ package collector import ( "errors" + "fmt" "sync" ) var errDiffNotEnabled = errors.New("diff not enabled") type DistinctValue[T comparable] struct { - values map[T]struct{} - new map[T]struct{} - len func(T) int - maxLen int - currLen int - limExceeded bool - diffEnabled bool - mtx sync.Mutex + values map[T]struct{} + new map[T]struct{} + len func(T) int + maxDataSize int + currDataSize int + currentValuesLen uint32 + maxValues uint32 + maxCacheHits uint32 + currentCacheHits uint32 + limExceeded bool + diffEnabled bool + stopReason string + mtx sync.Mutex } -// NewDistinctValue with the given maximum data size. This is calculated -// as the total length of the recorded strings. For ease of use, maximum=0 -// is interpreted as unlimited. +// NewDistinctValue with the given maximum data size and values limited. +// maxDataSize is calculated as the total length of the recorded strings. +// staleValueThreshold introduces a stop condition that is triggered when the number of found cache hits overcomes the limit +// For ease of use, maxDataSize=0 and maxValues are interpreted as unlimited. // Use NewDistinctValueWithDiff to enable diff support, but that one is slightly slower. -func NewDistinctValue[T comparable](maxDataSize int, len func(T) int) *DistinctValue[T] { +func NewDistinctValue[T comparable](maxDataSize int, maxValues uint32, staleValueThreshold uint32, len func(T) int) *DistinctValue[T] { return &DistinctValue[T]{ - values: make(map[T]struct{}), - maxLen: maxDataSize, - diffEnabled: false, // disable diff to make it faster - len: len, + values: make(map[T]struct{}), + maxDataSize: maxDataSize, + diffEnabled: false, // disable diff to make it faster + len: len, + maxValues: maxValues, + maxCacheHits: staleValueThreshold, } } // NewDistinctValueWithDiff is like NewDistinctValue but with diff support enabled. -func NewDistinctValueWithDiff[T comparable](maxDataSize int, len func(T) int) *DistinctValue[T] { +func NewDistinctValueWithDiff[T comparable](maxDataSize int, maxValues uint32, staleValueThreshold uint32, len func(T) int) *DistinctValue[T] { return &DistinctValue[T]{ - values: make(map[T]struct{}), - new: make(map[T]struct{}), - maxLen: maxDataSize, - diffEnabled: true, - len: len, + values: make(map[T]struct{}), + new: make(map[T]struct{}), + maxDataSize: maxDataSize, + diffEnabled: true, + len: len, + maxValues: maxValues, + maxCacheHits: staleValueThreshold, } } @@ -52,28 +63,39 @@ func (d *DistinctValue[T]) Collect(v T) (exceeded bool) { if d.limExceeded { return true } - // Calculate length valueLen := d.len(v) - // Can it fit? - // note: we will stop adding values slightly before the limit is reached - if d.maxLen > 0 && d.currLen+valueLen >= d.maxLen { - // No, it can't fit + if d.maxDataSize > 0 && d.currDataSize+valueLen >= d.maxDataSize { + d.stopReason = fmt.Sprintf("Max data exceeded: dataSize %d, maxDataSize %d", d.currDataSize, d.maxDataSize) + d.limExceeded = true + return true + } + + if d.maxValues > 0 && d.currentValuesLen >= d.maxValues { + d.stopReason = fmt.Sprintf("Max values exceeded: values %d, maxValues %d", d.currentValuesLen, d.maxValues) d.limExceeded = true return true } + if d.maxCacheHits > 0 && d.currentCacheHits >= d.maxCacheHits { + d.stopReason = fmt.Sprintf("Max stale values exceeded: cacheHits %d, maxValues %d", d.currentValuesLen, d.maxCacheHits) + d.limExceeded = true + return true + } if _, ok := d.values[v]; ok { - return // Already present + d.currentCacheHits++ + return false // Already present } + d.currentCacheHits = 0 // CacheHits reset to 0 when a new value is found if d.diffEnabled { d.new[v] = struct{}{} } d.values[v] = struct{}{} - d.currLen += valueLen + d.currDataSize += valueLen + d.currentValuesLen++ return false } @@ -100,12 +122,16 @@ func (d *DistinctValue[T]) Exceeded() bool { return d.limExceeded } +func (d *DistinctValue[T]) StopReason() string { + return d.stopReason +} + // Size is the total size of all distinct items collected func (d *DistinctValue[T]) Size() int { d.mtx.Lock() defer d.mtx.Unlock() - return d.currLen + return d.currDataSize } // Diff returns all new strings collected since the last time diff was called diff --git a/pkg/collector/distinct_value_collector_test.go b/pkg/collector/distinct_value_collector_test.go index d0bd470db97..4856173836f 100644 --- a/pkg/collector/distinct_value_collector_test.go +++ b/pkg/collector/distinct_value_collector_test.go @@ -11,18 +11,28 @@ import ( ) func TestDistinctValueCollector(t *testing.T) { - d := NewDistinctValue[string](10, func(s string) int { return len(s) }) + d := NewDistinctValue(10, 0, 0, func(s string) int { return len(s) }) - var stop bool - stop = d.Collect("123") - require.False(t, stop) - stop = d.Collect("4567") - require.False(t, stop) - stop = d.Collect("890") - require.True(t, stop) + require.False(t, d.Collect("123")) + require.False(t, d.Collect("4567")) + require.True(t, d.Collect("890")) + require.True(t, d.Exceeded()) + stringsSlicesEqual(t, []string{"123", "4567"}, d.Values()) + + // diff fails when diff is not enabled + res, err := d.Diff() + require.Nil(t, res) + require.Error(t, err, errDiffNotEnabled) +} + +func TestDistinctValueCollectorWithMaxValuesLimited(t *testing.T) { + d := NewDistinctValue(0, 2, 0, func(s string) int { return len(s) }) + require.False(t, d.Collect("123")) + require.False(t, d.Collect("4567")) + require.True(t, d.Collect("890")) require.True(t, d.Exceeded()) - require.Equal(t, stop, d.Exceeded()) // final stop should be same as Exceeded + stringsSlicesEqual(t, []string{"123", "4567"}, d.Values()) // diff fails when diff is not enabled @@ -31,17 +41,34 @@ func TestDistinctValueCollector(t *testing.T) { require.Error(t, err, errDiffNotEnabled) } +func TestDistinctValueCollectorWithCacheHitsLimits(t *testing.T) { + d := NewDistinctValue(0, 0, 2, func(s string) int { return len(s) }) + require.False(t, d.Collect("123")) + require.False(t, d.Collect("4567")) + require.False(t, d.Collect("890")) + require.False(t, d.Collect("890")) + require.False(t, d.Collect("890")) + require.True(t, d.Collect("890")) + require.True(t, d.Exceeded()) + stringsSlicesEqual(t, []string{"123", "4567", "890"}, d.Values()) + + // diff fails when diff is not enabled + res, err := d.Diff() + require.Nil(t, res) + require.Error(t, err, errDiffNotEnabled) +} + func TestDistinctValueCollectorDiff(t *testing.T) { - d := NewDistinctValueWithDiff[string](0, func(s string) int { return len(s) }) + d := NewDistinctValueWithDiff(0, 0, 0, func(s string) int { return len(s) }) - d.Collect("123") - d.Collect("4567") + require.False(t, d.Collect("123")) + require.False(t, d.Collect("4567")) stringsSlicesEqual(t, []string{"123", "4567"}, readDistinctValueDiff(t, d)) stringsSlicesEqual(t, []string{}, readDistinctValueDiff(t, d)) - d.Collect("123") - d.Collect("890") + require.False(t, d.Collect("123")) + require.False(t, d.Collect("890")) stringsSlicesEqual(t, []string{"890"}, readDistinctValueDiff(t, d)) stringsSlicesEqual(t, []string{}, readDistinctValueDiff(t, d)) @@ -72,6 +99,7 @@ func BenchmarkDistinctValueCollect(b *testing.B) { Value: fmt.Sprintf("value_%d_%d", i, j), } } + ingesterTagValues[i] = tagValues } limits := []int{ @@ -85,7 +113,7 @@ func BenchmarkDistinctValueCollect(b *testing.B) { for _, lim := range limits { b.Run("uniques_limit:"+strconv.Itoa(lim), func(b *testing.B) { for n := 0; n < b.N; n++ { - distinctValues := NewDistinctValue(lim, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + distinctValues := NewDistinctValue(lim, 0, 0, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) for _, tagValues := range ingesterTagValues { for _, v := range tagValues { if distinctValues.Collect(v) { @@ -98,7 +126,7 @@ func BenchmarkDistinctValueCollect(b *testing.B) { b.Run("duplicates_limit:"+strconv.Itoa(lim), func(b *testing.B) { for n := 0; n < b.N; n++ { - distinctValues := NewDistinctValue(lim, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + distinctValues := NewDistinctValue(lim, 0, 0, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) for i := 0; i < numIngesters; i++ { for j := 0; j < numTagValuesPerIngester; j++ { // collect first item to simulate duplicates diff --git a/pkg/collector/scoped_distinct_string.go b/pkg/collector/scoped_distinct_string.go index abe902122c7..b5c2b487f4f 100644 --- a/pkg/collector/scoped_distinct_string.go +++ b/pkg/collector/scoped_distinct_string.go @@ -4,31 +4,48 @@ import ( "sync" ) +const IntrinsicScope = "intrinsic" + type ScopedDistinctString struct { - cols map[string]*DistinctString - newCol func(int) *DistinctString - maxLen int - curLen int - limExceeded bool - diffEnabled bool - mtx sync.Mutex + cols map[string]*DistinctString + newCol func(int, uint32, uint32) *DistinctString + maxDataSize int + currDataSize int + limExceeded bool + maxCacheHits uint32 + diffEnabled bool + maxTagsPerScope uint32 + stopReason string + mtx sync.Mutex } -func NewScopedDistinctString(maxDataSize int) *ScopedDistinctString { +// NewScopedDistinctString collects the tags per scope +// MaxDataSize is calculated as the total length of the recorded strings. +// MaxTagsPerScope controls how many tags can be added per scope. The intrinsic scope is unbounded. +// For ease of use, maxDataSize=0 and maxTagsPerScope=0 are interpreted as unlimited. +func NewScopedDistinctString(maxDataSize int, maxTagsPerScope uint32, staleValueThreshold uint32) *ScopedDistinctString { return &ScopedDistinctString{ - cols: map[string]*DistinctString{}, - newCol: NewDistinctString, - maxLen: maxDataSize, - diffEnabled: false, + cols: map[string]*DistinctString{}, + newCol: NewDistinctString, + maxDataSize: maxDataSize, + diffEnabled: false, + maxTagsPerScope: maxTagsPerScope, + maxCacheHits: staleValueThreshold, } } -func NewScopedDistinctStringWithDiff(maxDataSize int) *ScopedDistinctString { +// NewScopedDistinctStringWithDiff collects the tags per scope with diff +// MaxDataSize is calculated as the total length of the recorded strings. +// MaxTagsPerScope controls how many tags can be added per scope. The intrinsic scope is unbounded. +// For ease of use, maxDataSize=0 and maxTagsPerScope=0 are interpreted as unlimited. +func NewScopedDistinctStringWithDiff(maxDataSize int, maxTagsPerScope uint32, staleValueThreshold uint32) *ScopedDistinctString { return &ScopedDistinctString{ - cols: map[string]*DistinctString{}, - newCol: NewDistinctStringWithDiff, - maxLen: maxDataSize, - diffEnabled: true, + cols: map[string]*DistinctString{}, + newCol: NewDistinctStringWithDiff, + maxDataSize: maxDataSize, + diffEnabled: true, + maxTagsPerScope: maxTagsPerScope, + maxCacheHits: staleValueThreshold, } } @@ -45,7 +62,7 @@ func (d *ScopedDistinctString) Collect(scope string, val string) (exceeded bool) valueLen := len(val) // can it fit? - if d.maxLen > 0 && d.curLen+valueLen > d.maxLen { + if d.maxDataSize > 0 && d.currDataSize+valueLen > d.maxDataSize { // No d.limExceeded = true return true @@ -54,13 +71,23 @@ func (d *ScopedDistinctString) Collect(scope string, val string) (exceeded bool) // get or create collector col, ok := d.cols[scope] if !ok { - col = d.newCol(0) + if scope == IntrinsicScope { + col = d.newCol(0, 0, 0) + } else { + col = d.newCol(0, d.maxTagsPerScope, d.maxCacheHits) + } d.cols[scope] = col } // add valueLen if we successfully added the value if col.Collect(val) { - d.curLen += valueLen + d.currDataSize += valueLen + } + if col.Exceeded() { + // we stop if one of the scopes exceed the limit + d.limExceeded = true + d.stopReason = col.stopReason + return true } return false } @@ -80,11 +107,25 @@ func (d *ScopedDistinctString) Strings() map[string][]string { } // Exceeded indicates if some values were lost because the maximum size limit was met. +// Or because one of the scopes max tags was reached. func (d *ScopedDistinctString) Exceeded() bool { d.mtx.Lock() defer d.mtx.Unlock() - return d.limExceeded + if d.limExceeded { + return true + } + + for _, v := range d.cols { + if v.Exceeded() { + return true + } + } + return false +} + +func (d *ScopedDistinctString) StopReason() string { + return d.stopReason } // Diff returns all new strings collected since the last time Diff was called diff --git a/pkg/collector/scoped_distinct_string_test.go b/pkg/collector/scoped_distinct_string_test.go index 9b090cef442..7ae38c5ffb1 100644 --- a/pkg/collector/scoped_distinct_string_test.go +++ b/pkg/collector/scoped_distinct_string_test.go @@ -12,10 +12,12 @@ import ( func TestScopedDistinct(t *testing.T) { tcs := []struct { - in map[string][]string - expected map[string][]string - limit int - exceeded bool + in map[string][]string + expected map[string][]string + maxBytes int + maxItemsPerScope uint32 + staleValueThreshold uint32 + exceeded bool }{ { in: map[string][]string{ @@ -46,13 +48,27 @@ func TestScopedDistinct(t *testing.T) { "scope1": {"val1", "val2"}, "scope2": {"val1"}, }, - limit: 13, + maxBytes: 13, exceeded: true, }, + { + in: map[string][]string{ + "intrinsic": {"val1", "val2"}, + "scope2": {"val1", "val2"}, + "scope3": {"val1", "val2", "val3"}, + }, + expected: map[string][]string{ + "intrinsic": {"val1", "val2"}, + "scope2": {"val1"}, + }, + maxBytes: 0, + maxItemsPerScope: 1, + exceeded: true, + }, } for _, tc := range tcs { - c := NewScopedDistinctString(tc.limit) + c := NewScopedDistinctString(tc.maxBytes, tc.maxItemsPerScope, tc.staleValueThreshold) // get and sort keys so we can deterministically add values keys := []string{} @@ -61,18 +77,15 @@ func TestScopedDistinct(t *testing.T) { } slices.Sort(keys) - var stop bool for _, k := range keys { v := tc.in[k] for _, val := range v { - stop = c.Collect(k, val) + c.Collect(k, val) } } // check if we exceeded the limit, and Collect and Exceeded return the same value require.Equal(t, tc.exceeded, c.Exceeded()) - require.Equal(t, tc.exceeded, stop) - require.Equal(t, stop, c.Exceeded()) actual := c.Strings() assertMaps(t, tc.expected, actual) @@ -80,7 +93,7 @@ func TestScopedDistinct(t *testing.T) { } func TestScopedDistinctDiff(t *testing.T) { - c := NewScopedDistinctStringWithDiff(0) + c := NewScopedDistinctStringWithDiff(0, 0, 0) c.Collect("scope1", "val1") expected := map[string][]string{ @@ -122,7 +135,7 @@ func TestScopedDistinctDiff(t *testing.T) { assertMaps(t, map[string][]string{}, readScopedDistinctStringDiff(t, c)) // diff should error when diff is not enabled - col := NewScopedDistinctString(0) + col := NewScopedDistinctString(0, 0, 0) col.Collect("scope1", "val1") res, err := col.Diff() require.Nil(t, res) @@ -144,7 +157,7 @@ func assertMaps(t *testing.T, expected, actual map[string][]string) { } func TestScopedDistinctStringCollectorIsSafe(t *testing.T) { - d := NewScopedDistinctString(0) // no limit + d := NewScopedDistinctString(0, 0, 0) // no limit var wg sync.WaitGroup for i := 0; i < 10; i++ { @@ -195,7 +208,7 @@ func BenchmarkScopedDistinctStringCollect(b *testing.B) { for _, lim := range limits { b.Run("uniques_limit:"+strconv.Itoa(lim), func(b *testing.B) { for n := 0; n < b.N; n++ { - scopedDistinctStrings := NewScopedDistinctString(lim) + scopedDistinctStrings := NewScopedDistinctString(lim, 0, 0) for _, tags := range ingesterTags { for scope, values := range tags { for _, v := range values { @@ -210,7 +223,7 @@ func BenchmarkScopedDistinctStringCollect(b *testing.B) { b.Run("duplicates_limit:"+strconv.Itoa(lim), func(b *testing.B) { for n := 0; n < b.N; n++ { - scopedDistinctStrings := NewScopedDistinctString(lim) + scopedDistinctStrings := NewScopedDistinctString(lim, 0, 0) for i := 0; i < numIngesters; i++ { for scope := range ingesterTags[i] { // collect first item to simulate duplicates diff --git a/pkg/tempopb/tempo.pb.go b/pkg/tempopb/tempo.pb.go index 427add0ea36..de45413874f 100644 --- a/pkg/tempopb/tempo.pb.go +++ b/pkg/tempopb/tempo.pb.go @@ -1045,10 +1045,12 @@ func (m *SearchMetrics) GetInspectedSpans() uint64 { } type SearchTagsRequest struct { - Scope string `protobuf:"bytes,1,opt,name=scope,proto3" json:"scope,omitempty"` - Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` - Start uint32 `protobuf:"varint,3,opt,name=start,proto3" json:"start,omitempty"` - End uint32 `protobuf:"varint,4,opt,name=end,proto3" json:"end,omitempty"` + Scope string `protobuf:"bytes,1,opt,name=scope,proto3" json:"scope,omitempty"` + Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` + Start uint32 `protobuf:"varint,3,opt,name=start,proto3" json:"start,omitempty"` + End uint32 `protobuf:"varint,4,opt,name=end,proto3" json:"end,omitempty"` + MaxTagsPerScope uint32 `protobuf:"varint,5,opt,name=maxTagsPerScope,proto3" json:"maxTagsPerScope,omitempty"` + StaleValuesThreshold uint32 `protobuf:"varint,6,opt,name=staleValuesThreshold,proto3" json:"staleValuesThreshold,omitempty"` } func (m *SearchTagsRequest) Reset() { *m = SearchTagsRequest{} } @@ -1112,21 +1114,37 @@ func (m *SearchTagsRequest) GetEnd() uint32 { return 0 } +func (m *SearchTagsRequest) GetMaxTagsPerScope() uint32 { + if m != nil { + return m.MaxTagsPerScope + } + return 0 +} + +func (m *SearchTagsRequest) GetStaleValuesThreshold() uint32 { + if m != nil { + return m.StaleValuesThreshold + } + return 0 +} + // SearchTagsBlockRequest takes SearchTagsRequest parameters as well as all information necessary // to search a block in the backend. type SearchTagsBlockRequest struct { - SearchReq *SearchTagsRequest `protobuf:"bytes,1,opt,name=searchReq,proto3" json:"searchReq,omitempty"` - BlockID string `protobuf:"bytes,2,opt,name=blockID,proto3" json:"blockID,omitempty"` - StartPage uint32 `protobuf:"varint,3,opt,name=startPage,proto3" json:"startPage,omitempty"` - PagesToSearch uint32 `protobuf:"varint,4,opt,name=pagesToSearch,proto3" json:"pagesToSearch,omitempty"` - Encoding string `protobuf:"bytes,5,opt,name=encoding,proto3" json:"encoding,omitempty"` - IndexPageSize uint32 `protobuf:"varint,6,opt,name=indexPageSize,proto3" json:"indexPageSize,omitempty"` - TotalRecords uint32 `protobuf:"varint,7,opt,name=totalRecords,proto3" json:"totalRecords,omitempty"` - DataEncoding string `protobuf:"bytes,8,opt,name=dataEncoding,proto3" json:"dataEncoding,omitempty"` - Version string `protobuf:"bytes,9,opt,name=version,proto3" json:"version,omitempty"` - Size_ uint64 `protobuf:"varint,10,opt,name=size,proto3" json:"size,omitempty"` - FooterSize uint32 `protobuf:"varint,11,opt,name=footerSize,proto3" json:"footerSize,omitempty"` - DedicatedColumns []*DedicatedColumn `protobuf:"bytes,12,rep,name=dedicatedColumns,proto3" json:"dedicatedColumns,omitempty"` + SearchReq *SearchTagsRequest `protobuf:"bytes,1,opt,name=searchReq,proto3" json:"searchReq,omitempty"` + BlockID string `protobuf:"bytes,2,opt,name=blockID,proto3" json:"blockID,omitempty"` + StartPage uint32 `protobuf:"varint,3,opt,name=startPage,proto3" json:"startPage,omitempty"` + PagesToSearch uint32 `protobuf:"varint,4,opt,name=pagesToSearch,proto3" json:"pagesToSearch,omitempty"` + Encoding string `protobuf:"bytes,5,opt,name=encoding,proto3" json:"encoding,omitempty"` + IndexPageSize uint32 `protobuf:"varint,6,opt,name=indexPageSize,proto3" json:"indexPageSize,omitempty"` + TotalRecords uint32 `protobuf:"varint,7,opt,name=totalRecords,proto3" json:"totalRecords,omitempty"` + DataEncoding string `protobuf:"bytes,8,opt,name=dataEncoding,proto3" json:"dataEncoding,omitempty"` + Version string `protobuf:"bytes,9,opt,name=version,proto3" json:"version,omitempty"` + Size_ uint64 `protobuf:"varint,10,opt,name=size,proto3" json:"size,omitempty"` + FooterSize uint32 `protobuf:"varint,11,opt,name=footerSize,proto3" json:"footerSize,omitempty"` + DedicatedColumns []*DedicatedColumn `protobuf:"bytes,12,rep,name=dedicatedColumns,proto3" json:"dedicatedColumns,omitempty"` + MaxTagsPerScope uint32 `protobuf:"varint,13,opt,name=maxTagsPerScope,proto3" json:"maxTagsPerScope,omitempty"` + StaleValueThreshold uint32 `protobuf:"varint,14,opt,name=staleValueThreshold,proto3" json:"staleValueThreshold,omitempty"` } func (m *SearchTagsBlockRequest) Reset() { *m = SearchTagsBlockRequest{} } @@ -1246,6 +1264,20 @@ func (m *SearchTagsBlockRequest) GetDedicatedColumns() []*DedicatedColumn { return nil } +func (m *SearchTagsBlockRequest) GetMaxTagsPerScope() uint32 { + if m != nil { + return m.MaxTagsPerScope + } + return 0 +} + +func (m *SearchTagsBlockRequest) GetStaleValueThreshold() uint32 { + if m != nil { + return m.StaleValueThreshold + } + return 0 +} + type SearchTagValuesBlockRequest struct { SearchReq *SearchTagValuesRequest `protobuf:"bytes,1,opt,name=searchReq,proto3" json:"searchReq,omitempty"` BlockID string `protobuf:"bytes,2,opt,name=blockID,proto3" json:"blockID,omitempty"` @@ -1535,10 +1567,12 @@ func (m *SearchTagsV2Scope) GetTags() []string { } type SearchTagValuesRequest struct { - TagName string `protobuf:"bytes,1,opt,name=tagName,proto3" json:"tagName,omitempty"` - Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` - Start uint32 `protobuf:"varint,4,opt,name=start,proto3" json:"start,omitempty"` - End uint32 `protobuf:"varint,5,opt,name=end,proto3" json:"end,omitempty"` + TagName string `protobuf:"bytes,1,opt,name=tagName,proto3" json:"tagName,omitempty"` + Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` + Start uint32 `protobuf:"varint,4,opt,name=start,proto3" json:"start,omitempty"` + End uint32 `protobuf:"varint,5,opt,name=end,proto3" json:"end,omitempty"` + MaxTagValues uint32 `protobuf:"varint,6,opt,name=maxTagValues,proto3" json:"maxTagValues,omitempty"` + StaleValueThreshold uint32 `protobuf:"varint,7,opt,name=staleValueThreshold,proto3" json:"staleValueThreshold,omitempty"` } func (m *SearchTagValuesRequest) Reset() { *m = SearchTagValuesRequest{} } @@ -1602,6 +1636,20 @@ func (m *SearchTagValuesRequest) GetEnd() uint32 { return 0 } +func (m *SearchTagValuesRequest) GetMaxTagValues() uint32 { + if m != nil { + return m.MaxTagValues + } + return 0 +} + +func (m *SearchTagValuesRequest) GetStaleValueThreshold() uint32 { + if m != nil { + return m.StaleValueThreshold + } + return 0 +} + type SearchTagValuesResponse struct { TagValues []string `protobuf:"bytes,1,rep,name=tagValues,proto3" json:"tagValues,omitempty"` Metrics *MetadataMetrics `protobuf:"bytes,2,opt,name=metrics,proto3" json:"metrics,omitempty"` @@ -3539,186 +3587,191 @@ func init() { func init() { proto.RegisterFile("pkg/tempopb/tempo.proto", fileDescriptor_f22805646f4f62b6) } var fileDescriptor_f22805646f4f62b6 = []byte{ - // 2855 bytes of a gzipped FileDescriptorProto + // 2935 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0x4b, 0x6f, 0x23, 0xc7, - 0xd1, 0x1a, 0xf1, 0x5d, 0x24, 0x25, 0xaa, 0x57, 0x96, 0xb9, 0xdc, 0xb5, 0x24, 0x8f, 0x17, 0x86, - 0x3e, 0x3f, 0x28, 0x2d, 0xbd, 0xc6, 0xe7, 0xb5, 0x13, 0x07, 0xd2, 0x8a, 0x59, 0xcb, 0xd6, 0xcb, - 0x4d, 0x5a, 0x36, 0x02, 0x03, 0xc2, 0x88, 0xec, 0xe5, 0x0e, 0x44, 0xce, 0xd0, 0x33, 0x43, 0x79, - 0x99, 0x83, 0x91, 0x04, 0xc8, 0x21, 0x40, 0x0e, 0x39, 0x24, 0x87, 0xfc, 0x82, 0x20, 0xa7, 0x1c, - 0x92, 0x7f, 0x10, 0x20, 0x70, 0x10, 0xc4, 0x30, 0x90, 0x8b, 0x91, 0x83, 0x11, 0xd8, 0x87, 0xe4, - 0x67, 0x04, 0x5d, 0xdd, 0x3d, 0x6f, 0x49, 0x5e, 0xef, 0x1a, 0xf1, 0xc1, 0x27, 0x75, 0xd5, 0x54, - 0x57, 0x57, 0xd7, 0xbb, 0x9a, 0x82, 0x27, 0xc7, 0xa7, 0x83, 0x75, 0x8f, 0x8d, 0xc6, 0xf6, 0xf8, - 0x44, 0xfc, 0x6d, 0x8e, 0x1d, 0xdb, 0xb3, 0x49, 0x41, 0x22, 0x1b, 0x4b, 0x3d, 0x7b, 0x34, 0xb2, - 0xad, 0xf5, 0xb3, 0x9b, 0xeb, 0x62, 0x25, 0x08, 0x1a, 0x2f, 0x0e, 0x4c, 0xef, 0xfe, 0xe4, 0xa4, - 0xd9, 0xb3, 0x47, 0xeb, 0x03, 0x7b, 0x60, 0xaf, 0x23, 0xfa, 0x64, 0x72, 0x0f, 0x21, 0x04, 0x70, - 0x25, 0xc9, 0x17, 0x3d, 0xc7, 0xe8, 0x31, 0xce, 0x05, 0x17, 0x02, 0xab, 0xff, 0x49, 0x83, 0x5a, - 0x97, 0xc3, 0x5b, 0xd3, 0x9d, 0x6d, 0xca, 0x3e, 0x98, 0x30, 0xd7, 0x23, 0x75, 0x28, 0x20, 0xcd, - 0xce, 0x76, 0x5d, 0x5b, 0xd5, 0xd6, 0x2a, 0x54, 0x81, 0x64, 0x19, 0xe0, 0x64, 0x68, 0xf7, 0x4e, - 0x3b, 0x9e, 0xe1, 0x78, 0xf5, 0xd9, 0x55, 0x6d, 0xad, 0x44, 0x43, 0x18, 0xd2, 0x80, 0x22, 0x42, - 0x6d, 0xab, 0x5f, 0xcf, 0xe0, 0x57, 0x1f, 0x26, 0xd7, 0xa1, 0xf4, 0xc1, 0x84, 0x39, 0xd3, 0x3d, - 0xbb, 0xcf, 0xea, 0x39, 0xfc, 0x18, 0x20, 0xc8, 0x0b, 0xb0, 0x60, 0x0c, 0x87, 0xf6, 0x87, 0x87, - 0x86, 0xe3, 0x99, 0xc6, 0x10, 0x65, 0xaa, 0xe7, 0x57, 0xb5, 0xb5, 0x22, 0x4d, 0x7e, 0xd0, 0xff, - 0xa3, 0xc1, 0x42, 0x48, 0x6c, 0x77, 0x6c, 0x5b, 0x2e, 0x23, 0x37, 0x20, 0x87, 0x82, 0xa2, 0xd4, - 0xe5, 0xd6, 0x5c, 0x53, 0xaa, 0xb0, 0x89, 0xa4, 0x54, 0x7c, 0x24, 0x2f, 0x41, 0x61, 0xc4, 0x3c, - 0xc7, 0xec, 0xb9, 0x78, 0x81, 0x72, 0xeb, 0x6a, 0x94, 0x8e, 0xb3, 0xdc, 0x13, 0x04, 0x54, 0x51, - 0x92, 0xdb, 0x90, 0x77, 0x3d, 0xc3, 0x9b, 0xb8, 0x78, 0xad, 0xb9, 0xd6, 0xd3, 0xc9, 0x3d, 0x4a, - 0x8c, 0x66, 0x07, 0x09, 0xa9, 0xdc, 0xc0, 0xb5, 0x39, 0x62, 0xae, 0x6b, 0x0c, 0x58, 0x3d, 0x8b, - 0xb7, 0x56, 0xa0, 0xfe, 0x0c, 0xe4, 0x05, 0x2d, 0xa9, 0x40, 0xf1, 0xce, 0xc1, 0xde, 0xe1, 0x6e, - 0xbb, 0xdb, 0xae, 0xcd, 0x90, 0x32, 0x14, 0x0e, 0x37, 0x69, 0x77, 0x67, 0x73, 0xb7, 0xa6, 0xe9, - 0x24, 0x64, 0x20, 0x29, 0x96, 0xfe, 0xc9, 0x2c, 0x54, 0x3b, 0xcc, 0x70, 0x7a, 0xf7, 0x95, 0xc9, - 0x5e, 0x85, 0x6c, 0xd7, 0x18, 0xb8, 0x75, 0x6d, 0x35, 0xb3, 0x56, 0x6e, 0xad, 0xfa, 0xd2, 0x45, - 0xa8, 0x9a, 0x9c, 0xa4, 0x6d, 0x79, 0xce, 0x74, 0x2b, 0xfb, 0xf1, 0xe7, 0x2b, 0x33, 0x14, 0xf7, - 0x90, 0x1b, 0x50, 0xdd, 0x33, 0xad, 0xed, 0x89, 0x63, 0x78, 0xa6, 0x6d, 0xed, 0x09, 0xb5, 0x54, - 0x69, 0x14, 0x89, 0x54, 0xc6, 0x83, 0x10, 0x55, 0x46, 0x52, 0x85, 0x91, 0x64, 0x11, 0x72, 0xbb, - 0xe6, 0xc8, 0xf4, 0xf0, 0xaa, 0x55, 0x2a, 0x00, 0x8e, 0x75, 0xd1, 0x63, 0x72, 0x02, 0x8b, 0x00, - 0xa9, 0x41, 0x86, 0x59, 0x7d, 0x34, 0x72, 0x95, 0xf2, 0x25, 0xa7, 0x7b, 0x9b, 0x7b, 0x44, 0xbd, - 0x88, 0x8a, 0x12, 0x00, 0x59, 0x83, 0xf9, 0xce, 0xd8, 0xb0, 0xdc, 0x43, 0xe6, 0xf0, 0xbf, 0x1d, - 0xe6, 0xd5, 0x4b, 0xb8, 0x27, 0x8e, 0x6e, 0xfc, 0x3f, 0x94, 0xfc, 0x2b, 0x72, 0xf6, 0xa7, 0x6c, - 0x8a, 0xbe, 0x50, 0xa2, 0x7c, 0xc9, 0xd9, 0x9f, 0x19, 0xc3, 0x09, 0x93, 0x8e, 0x2b, 0x80, 0x57, - 0x67, 0x5f, 0xd1, 0xf4, 0xbf, 0x64, 0x80, 0x08, 0x55, 0x6d, 0x71, 0x77, 0x55, 0x5a, 0xbd, 0x05, - 0x25, 0x57, 0x29, 0x50, 0x3a, 0xd5, 0x52, 0xba, 0x6a, 0x69, 0x40, 0xc8, 0x0d, 0x8e, 0x4e, 0xbf, - 0xb3, 0x2d, 0x0f, 0x52, 0x20, 0x0f, 0x01, 0xbc, 0xfa, 0x21, 0x77, 0x06, 0xa1, 0xbf, 0x00, 0xc1, - 0x35, 0x3c, 0x36, 0x06, 0xcc, 0xed, 0xda, 0x82, 0xb5, 0xd4, 0x61, 0x14, 0xc9, 0x43, 0x8c, 0x59, - 0x3d, 0xbb, 0x6f, 0x5a, 0x03, 0x19, 0x45, 0x3e, 0xcc, 0x39, 0x98, 0x56, 0x9f, 0x3d, 0xe0, 0xec, - 0x3a, 0xe6, 0x8f, 0x99, 0xd4, 0x6d, 0x14, 0x49, 0x74, 0xa8, 0x78, 0xb6, 0x67, 0x0c, 0x29, 0xeb, - 0xd9, 0x4e, 0xdf, 0xad, 0x17, 0x90, 0x28, 0x82, 0xe3, 0x34, 0x7d, 0xc3, 0x33, 0xda, 0xea, 0x24, - 0x61, 0x90, 0x08, 0x8e, 0xdf, 0xf3, 0x8c, 0x39, 0xae, 0x69, 0x5b, 0x68, 0x8f, 0x12, 0x55, 0x20, - 0x21, 0x90, 0x75, 0xf9, 0xf1, 0xb0, 0xaa, 0xad, 0x65, 0x29, 0xae, 0x79, 0xea, 0xb8, 0x67, 0xdb, - 0x1e, 0x73, 0x50, 0xb0, 0x32, 0x9e, 0x19, 0xc2, 0x90, 0x6d, 0xa8, 0xf5, 0x59, 0xdf, 0xec, 0x19, - 0x1e, 0xeb, 0xdf, 0xb1, 0x87, 0x93, 0x91, 0xe5, 0xd6, 0x2b, 0xe8, 0xcd, 0x75, 0x5f, 0xe5, 0xdb, - 0x51, 0x02, 0x9a, 0xd8, 0xa1, 0xff, 0x59, 0x83, 0xf9, 0x18, 0x15, 0xb9, 0x05, 0x39, 0xb7, 0x67, - 0x8f, 0x99, 0x0c, 0xdd, 0xe5, 0xf3, 0xd8, 0x35, 0x3b, 0x9c, 0x8a, 0x0a, 0x62, 0x7e, 0x07, 0xcb, - 0x18, 0x29, 0x5f, 0xc1, 0x35, 0xb9, 0x09, 0x59, 0x6f, 0x3a, 0x16, 0xf9, 0x65, 0xae, 0xf5, 0xd4, - 0xb9, 0x8c, 0xba, 0xd3, 0x31, 0xa3, 0x48, 0xaa, 0xaf, 0x40, 0x0e, 0xd9, 0x92, 0x22, 0x64, 0x3b, - 0x87, 0x9b, 0xfb, 0xb5, 0x19, 0x1e, 0xec, 0xb4, 0xdd, 0x39, 0x78, 0x87, 0xde, 0x69, 0x63, 0x7c, - 0x67, 0x39, 0x39, 0x01, 0xc8, 0x77, 0xba, 0x74, 0x67, 0xff, 0x6e, 0x6d, 0x46, 0x7f, 0x00, 0x73, - 0xca, 0xbb, 0x64, 0x6a, 0xbb, 0x05, 0x79, 0xcc, 0x5e, 0x2a, 0xc2, 0xaf, 0x47, 0xf3, 0x8f, 0xa0, - 0xde, 0x63, 0x9e, 0xc1, 0x2d, 0x44, 0x25, 0x2d, 0xd9, 0x88, 0xa7, 0xba, 0xb8, 0xf7, 0xc6, 0xf3, - 0x9c, 0xfe, 0x8f, 0x0c, 0x5c, 0x49, 0xe1, 0x18, 0x2f, 0x09, 0xa5, 0xa0, 0x24, 0xac, 0xc1, 0xbc, - 0x63, 0xdb, 0x5e, 0x87, 0x39, 0x67, 0x66, 0x8f, 0xed, 0x07, 0x2a, 0x8b, 0xa3, 0xb9, 0x77, 0x72, - 0x14, 0xb2, 0x47, 0x3a, 0x51, 0x21, 0xa2, 0x48, 0x5e, 0x08, 0x30, 0x24, 0xba, 0xe6, 0x88, 0xbd, - 0x63, 0x99, 0x0f, 0xf6, 0x0d, 0xcb, 0xc6, 0x48, 0xc8, 0xd2, 0xe4, 0x07, 0xee, 0x55, 0xfd, 0x20, - 0x25, 0x89, 0xf4, 0x12, 0xc2, 0x90, 0xe7, 0xa0, 0xe0, 0xca, 0x9c, 0x91, 0x47, 0x0d, 0xd4, 0x02, - 0x0d, 0x08, 0x3c, 0x55, 0x04, 0xe4, 0x05, 0x28, 0xca, 0x25, 0x8f, 0x89, 0x4c, 0x2a, 0xb1, 0x4f, - 0x41, 0x28, 0x54, 0x5c, 0x71, 0x39, 0x9e, 0xc3, 0xdd, 0x7a, 0x11, 0x77, 0x34, 0x2f, 0xb2, 0x4b, - 0xb3, 0x13, 0xda, 0x80, 0x49, 0x8a, 0x46, 0x78, 0x34, 0x8e, 0x60, 0x21, 0x41, 0x92, 0x92, 0xc7, - 0x9e, 0x0f, 0xe7, 0xb1, 0x72, 0xeb, 0x89, 0x90, 0x51, 0x83, 0xcd, 0xe1, 0xf4, 0xb6, 0x0b, 0x95, - 0xf0, 0x27, 0xcc, 0x43, 0x63, 0xc3, 0xba, 0x63, 0x4f, 0x2c, 0x0f, 0x19, 0xf3, 0x3c, 0xa4, 0x10, - 0x5c, 0xa7, 0xcc, 0x71, 0x6c, 0x47, 0x7c, 0x16, 0xc5, 0x20, 0x84, 0xd1, 0x7f, 0xae, 0x41, 0x41, - 0xea, 0x83, 0x3c, 0x03, 0x39, 0xbe, 0x51, 0xb9, 0x65, 0x35, 0xa2, 0x30, 0x2a, 0xbe, 0x61, 0x05, - 0x34, 0xbc, 0xde, 0x7d, 0xd6, 0x97, 0xdc, 0x14, 0x48, 0x5e, 0x03, 0x30, 0x3c, 0xcf, 0x31, 0x4f, - 0x26, 0x1e, 0xe3, 0x15, 0x85, 0xf3, 0xb8, 0xe6, 0xf3, 0x90, 0xed, 0xce, 0xd9, 0xcd, 0xe6, 0x5b, - 0x6c, 0x7a, 0xc4, 0x6f, 0x43, 0x43, 0xe4, 0x3c, 0xd6, 0xb3, 0xfc, 0x18, 0xb2, 0x04, 0x79, 0x7e, - 0x90, 0xef, 0x9b, 0x12, 0x4a, 0x0d, 0xe1, 0x54, 0xf7, 0xca, 0x9c, 0xe7, 0x5e, 0x37, 0xa0, 0xaa, - 0x9c, 0x89, 0xc3, 0xae, 0x74, 0xc4, 0x28, 0x32, 0x76, 0x8b, 0xdc, 0xc3, 0xdd, 0xe2, 0xb7, 0x7e, - 0x2d, 0x97, 0xc1, 0xc8, 0x23, 0xca, 0xb4, 0xdc, 0x31, 0xeb, 0x79, 0xac, 0xdf, 0x55, 0x41, 0x8f, - 0xf5, 0x2e, 0x86, 0x26, 0xcf, 0xc2, 0x9c, 0x8f, 0xda, 0x9a, 0xf2, 0xc3, 0x67, 0x51, 0xbe, 0x18, - 0x96, 0xac, 0x42, 0x19, 0xb3, 0x3b, 0x16, 0x37, 0x55, 0xb9, 0xc3, 0x28, 0x7e, 0xd1, 0x9e, 0x3d, - 0x1a, 0x0f, 0x99, 0xc7, 0xfa, 0x6f, 0xda, 0x27, 0xae, 0xaa, 0x3d, 0x11, 0x24, 0xf7, 0x1b, 0xdc, - 0x84, 0x14, 0x22, 0xd8, 0x02, 0x04, 0x97, 0x3b, 0x60, 0x29, 0xc4, 0xc9, 0xa3, 0x38, 0x71, 0x74, - 0x44, 0x6e, 0xac, 0xe1, 0x58, 0x83, 0xc2, 0x72, 0x23, 0x56, 0x1f, 0xf0, 0x78, 0xe0, 0xaa, 0xe1, - 0x55, 0x5d, 0x15, 0xe5, 0x45, 0x95, 0xce, 0x85, 0xb1, 0x65, 0xba, 0x5e, 0x84, 0x1c, 0x36, 0x93, - 0xaa, 0xb6, 0x23, 0x10, 0x34, 0x1e, 0x99, 0x94, 0xc6, 0x23, 0xeb, 0x37, 0x1e, 0xfa, 0x27, 0x19, - 0x58, 0x0a, 0x4e, 0x8a, 0xf4, 0x00, 0xaf, 0x24, 0x7b, 0x80, 0x46, 0x2c, 0x8b, 0x86, 0xa4, 0xfb, - 0xae, 0x0f, 0xf8, 0x76, 0xf4, 0x01, 0x9f, 0x65, 0xe0, 0x9a, 0x6f, 0x1c, 0x0c, 0xba, 0xa8, 0x55, - 0xbf, 0x9f, 0xb4, 0xea, 0x4a, 0xd2, 0xaa, 0x62, 0xe3, 0x77, 0xa6, 0xfd, 0x56, 0x99, 0xb6, 0xaf, - 0x5a, 0x75, 0x11, 0x76, 0xb2, 0x41, 0x6a, 0x40, 0xd1, 0x33, 0x06, 0xbc, 0x83, 0x10, 0xb5, 0xa8, - 0x44, 0x7d, 0x98, 0xb4, 0xe2, 0x6d, 0x50, 0x70, 0x9c, 0x2a, 0xcd, 0x89, 0x46, 0xe8, 0x23, 0x58, - 0x0c, 0x4e, 0x39, 0x6a, 0xf9, 0xe7, 0xb4, 0x20, 0x8f, 0x09, 0x47, 0x55, 0xbc, 0xb4, 0x5c, 0x70, - 0xd4, 0x12, 0x9d, 0xa4, 0xa4, 0xfc, 0x5a, 0xe7, 0xbf, 0x16, 0x4e, 0x7d, 0x92, 0xa1, 0x5f, 0xd0, - 0xb4, 0x50, 0x41, 0x23, 0x90, 0xf5, 0xf8, 0xe4, 0x37, 0x8b, 0x97, 0xc6, 0xb5, 0x3e, 0x0e, 0x65, - 0xb3, 0x88, 0x0f, 0x63, 0x1f, 0x27, 0xd4, 0xe2, 0xf7, 0x71, 0x02, 0xbc, 0x2c, 0x81, 0x66, 0x53, - 0x12, 0x68, 0x2e, 0x48, 0xa0, 0xa7, 0xf0, 0x64, 0xe2, 0x44, 0xa9, 0x31, 0x5e, 0x34, 0x14, 0x52, - 0x9a, 0x26, 0x40, 0x7c, 0x2d, 0xdd, 0xdc, 0x82, 0xa2, 0x3a, 0x06, 0xaf, 0x3f, 0xf5, 0x8b, 0x01, - 0xae, 0xd3, 0xe7, 0x3c, 0xfd, 0x27, 0x1a, 0x5c, 0x8d, 0xc9, 0x18, 0xb2, 0xeb, 0x7a, 0x5c, 0xca, - 0x72, 0x6b, 0x21, 0xe8, 0xe5, 0xe4, 0x97, 0x47, 0x15, 0xfc, 0xaf, 0x1a, 0xcc, 0xc7, 0x3e, 0xa6, - 0xd4, 0x70, 0x2d, 0xb5, 0x86, 0x47, 0x6a, 0xef, 0x6c, 0xbc, 0xf6, 0x26, 0xea, 0x77, 0x26, 0xad, - 0x7e, 0xc7, 0xfa, 0x80, 0x6c, 0xb2, 0x0f, 0x48, 0xa9, 0xe1, 0xb9, 0xd4, 0x1a, 0xae, 0xef, 0x43, - 0x0e, 0xbb, 0x10, 0xd2, 0x86, 0xaa, 0xc3, 0x5c, 0x7b, 0xe2, 0xf4, 0x58, 0x27, 0xd4, 0x0a, 0x06, - 0xe9, 0x54, 0xbc, 0x37, 0x9d, 0xdd, 0x6c, 0xd2, 0x30, 0x19, 0x8d, 0xee, 0xd2, 0xf7, 0xa1, 0x72, - 0x38, 0x71, 0x83, 0x89, 0xe7, 0x75, 0xa8, 0x62, 0xcf, 0xe9, 0x6e, 0x4d, 0xbb, 0xf2, 0x51, 0x27, - 0xb3, 0x36, 0x17, 0xd2, 0x32, 0xa7, 0x6e, 0x73, 0x0a, 0xca, 0x0c, 0xd7, 0xb6, 0x68, 0x94, 0x5c, - 0xef, 0x40, 0x8d, 0x53, 0xa0, 0xb0, 0xca, 0xfb, 0x5f, 0xf4, 0xa7, 0x28, 0x1e, 0x2d, 0x95, 0xad, - 0x27, 0x3e, 0xfe, 0x7c, 0x65, 0xe6, 0x9f, 0x9f, 0xaf, 0x54, 0x0f, 0x1d, 0x66, 0x0c, 0x87, 0x76, - 0x4f, 0x50, 0xab, 0xf1, 0xa9, 0x06, 0x19, 0xb3, 0x2f, 0xda, 0xd2, 0x0a, 0xe5, 0x4b, 0x7d, 0x4f, - 0x30, 0x15, 0x17, 0x90, 0x4c, 0x6f, 0x43, 0xe1, 0x04, 0xdb, 0xd9, 0xaf, 0x7c, 0x73, 0x45, 0xaf, - 0xdf, 0x00, 0x90, 0x6f, 0x3b, 0xdc, 0xc2, 0x4b, 0x91, 0x19, 0xaf, 0xa2, 0xc4, 0xd0, 0x5f, 0x87, - 0xd2, 0xae, 0x69, 0x9d, 0x76, 0x86, 0x66, 0x8f, 0x8f, 0xa0, 0xb9, 0xa1, 0x69, 0x9d, 0xaa, 0xb3, - 0xae, 0x25, 0xcf, 0xe2, 0x67, 0x34, 0xf9, 0x06, 0x2a, 0x28, 0xf5, 0x9f, 0x69, 0x40, 0x38, 0x52, - 0xb9, 0x63, 0xd0, 0x47, 0x89, 0x80, 0xd7, 0xc2, 0x01, 0x5f, 0x87, 0xc2, 0xc0, 0xb1, 0x27, 0xe3, - 0x2d, 0x95, 0x08, 0x14, 0xc8, 0xe9, 0x87, 0xf8, 0xb4, 0x23, 0xba, 0x65, 0x01, 0x7c, 0xe5, 0x04, - 0xf1, 0x0b, 0x1e, 0x7d, 0x81, 0x10, 0x9d, 0xc9, 0x68, 0x64, 0x38, 0xd3, 0xff, 0x8d, 0x2c, 0xbf, - 0xd7, 0xe0, 0x4a, 0x44, 0x21, 0x41, 0xa6, 0x62, 0xae, 0x67, 0x8e, 0x78, 0xb5, 0x41, 0x49, 0x8a, - 0x34, 0x40, 0x44, 0x87, 0x26, 0xd1, 0x67, 0x87, 0x86, 0xa6, 0x67, 0x61, 0x0e, 0xfd, 0xaf, 0xe3, - 0x93, 0x08, 0xd1, 0x62, 0x58, 0xd2, 0x0c, 0xd2, 0x46, 0x16, 0x2d, 0xb8, 0x18, 0x19, 0x99, 0x12, - 0x29, 0xe3, 0x7b, 0x50, 0xa1, 0xc6, 0x87, 0x6f, 0x98, 0xae, 0x67, 0x0f, 0x1c, 0x63, 0xc4, 0x9d, - 0xe4, 0x64, 0xd2, 0x3b, 0x65, 0x9e, 0x4c, 0x13, 0x12, 0xe2, 0x77, 0xef, 0x85, 0x24, 0x13, 0x80, - 0xfe, 0x26, 0x14, 0xd5, 0xd0, 0x91, 0x32, 0x47, 0xbe, 0x10, 0x9d, 0x23, 0x97, 0xa2, 0xb3, 0xeb, - 0xdb, 0xbb, 0x7c, 0x58, 0x34, 0x7b, 0x2a, 0x7f, 0xfe, 0x5a, 0x83, 0x72, 0x48, 0x44, 0xb2, 0x05, - 0x0b, 0x43, 0xc3, 0x63, 0x56, 0x6f, 0x7a, 0x7c, 0x5f, 0x89, 0x27, 0xbd, 0x32, 0x98, 0x48, 0xc3, - 0xb2, 0xd3, 0x9a, 0xa4, 0x0f, 0x6e, 0xf3, 0x7f, 0x90, 0x77, 0x99, 0x63, 0xca, 0x80, 0x0c, 0xa7, - 0x5c, 0x7f, 0x56, 0x92, 0x04, 0xfc, 0xe2, 0x22, 0xc0, 0xa5, 0x62, 0x25, 0xa4, 0xff, 0x3d, 0xea, - 0xdd, 0xd2, 0xb1, 0x92, 0x23, 0xee, 0x25, 0xd6, 0x9a, 0x4d, 0xb5, 0x56, 0x20, 0x5f, 0xe6, 0x32, - 0xf9, 0x6a, 0x90, 0x19, 0xdf, 0xbe, 0x2d, 0x07, 0x44, 0xbe, 0x14, 0x98, 0x97, 0x65, 0xfe, 0xe4, - 0x4b, 0x81, 0xd9, 0x90, 0x53, 0x11, 0x5f, 0x22, 0xe6, 0xe5, 0x0d, 0x39, 0xfe, 0xf0, 0xa5, 0xfe, - 0x2e, 0x34, 0xd2, 0xe2, 0x44, 0xba, 0xe8, 0x6d, 0x28, 0xb9, 0x88, 0x32, 0x59, 0x32, 0x05, 0xa4, - 0xec, 0x0b, 0xa8, 0xf5, 0xdf, 0x68, 0x50, 0x8d, 0x18, 0x36, 0x52, 0x3b, 0x73, 0xb2, 0x76, 0x56, - 0x40, 0xb3, 0x50, 0x19, 0x19, 0xaa, 0x59, 0x1c, 0xba, 0x87, 0xfa, 0xd6, 0xa8, 0x76, 0x8f, 0x43, - 0xae, 0x7c, 0xc3, 0xd6, 0x5c, 0x0e, 0x9d, 0xe0, 0xe5, 0x8a, 0x54, 0x3b, 0xe1, 0x50, 0x5f, 0x5e, - 0x4c, 0xeb, 0xe3, 0x44, 0x2e, 0x9e, 0xcb, 0x0b, 0xc8, 0x5b, 0xbd, 0x85, 0x13, 0xc8, 0x9e, 0x9a, - 0x56, 0x1f, 0x7b, 0xcd, 0x1c, 0xc5, 0xb5, 0xce, 0xc4, 0xf3, 0xae, 0x14, 0x7c, 0xdb, 0xf0, 0x0c, - 0xde, 0x48, 0x3a, 0xcc, 0x9d, 0x0c, 0xbd, 0x6e, 0x50, 0xda, 0x43, 0x18, 0xde, 0x84, 0x09, 0x48, - 0xba, 0x4d, 0x23, 0x35, 0x86, 0x90, 0x82, 0x4a, 0x4a, 0x9e, 0x05, 0x17, 0x12, 0x5f, 0xb9, 0x9b, - 0x0c, 0x8d, 0x13, 0x36, 0x0c, 0x75, 0x44, 0x01, 0x82, 0xcb, 0x81, 0xc0, 0x51, 0xa8, 0x9b, 0x08, - 0x61, 0xc8, 0x3a, 0xcc, 0x7a, 0xca, 0x35, 0x56, 0xce, 0x97, 0xe1, 0xd0, 0x36, 0x2d, 0x8f, 0xce, - 0x7a, 0x2e, 0x8f, 0xa1, 0xa5, 0xf4, 0xcf, 0x68, 0x0c, 0x53, 0x0a, 0x51, 0xa5, 0xb8, 0xe6, 0xde, - 0x71, 0x66, 0x0c, 0xf1, 0x60, 0x8d, 0xf2, 0x25, 0xaf, 0xcf, 0xec, 0x01, 0x1b, 0x8d, 0x87, 0x86, - 0xd3, 0x95, 0xef, 0x71, 0x19, 0xfc, 0x89, 0x26, 0x8e, 0x26, 0xcf, 0x41, 0x4d, 0xa1, 0xd4, 0xfb, - 0xbc, 0x74, 0xce, 0x04, 0x5e, 0xef, 0xc0, 0x15, 0x7c, 0x6a, 0xdf, 0xb1, 0x5c, 0xcf, 0xb0, 0xbc, - 0x8b, 0xb3, 0xb2, 0x9f, 0x65, 0x65, 0xa6, 0x89, 0x64, 0x59, 0x11, 0x9b, 0x98, 0x65, 0x1f, 0xc0, - 0x62, 0x94, 0xa9, 0x74, 0xe1, 0xa6, 0x1f, 0x53, 0xc2, 0x7f, 0x83, 0xb4, 0x23, 0x29, 0x3b, 0xf8, - 0xd5, 0x0f, 0xac, 0x87, 0x7f, 0xc4, 0xfc, 0xa9, 0x06, 0xd5, 0x08, 0x2f, 0x72, 0x1b, 0xf2, 0x68, - 0xb6, 0x64, 0xcc, 0x24, 0x5f, 0x67, 0xe4, 0x6f, 0x23, 0x72, 0x43, 0xb4, 0x99, 0xd4, 0x64, 0x32, - 0x24, 0x2b, 0x50, 0x1e, 0x3b, 0xf6, 0xe8, 0x58, 0x72, 0x15, 0x2f, 0x99, 0xc0, 0x51, 0xbb, 0x88, - 0xd1, 0xff, 0x90, 0x81, 0x05, 0xbc, 0x3e, 0x35, 0xac, 0x01, 0x7b, 0x2c, 0x1a, 0xc5, 0x99, 0xcb, - 0x63, 0x63, 0x69, 0x46, 0x5c, 0x47, 0x7f, 0x55, 0x2b, 0xc4, 0x7f, 0x55, 0x0b, 0xcd, 0xa9, 0xc5, - 0x0b, 0xe6, 0xd4, 0xd2, 0xa5, 0x73, 0x2a, 0xa4, 0xcd, 0xa9, 0xa1, 0xe9, 0xb0, 0x1c, 0x9d, 0x0e, - 0xc3, 0x13, 0x6c, 0x25, 0x36, 0xc1, 0xaa, 0xc9, 0xb1, 0x7a, 0xee, 0xe4, 0x38, 0xf7, 0x95, 0x26, - 0xc7, 0xf9, 0x87, 0x9d, 0x1c, 0xb1, 0xbe, 0x4b, 0xd7, 0x77, 0xeb, 0x35, 0x71, 0x67, 0x1f, 0xa1, - 0xbb, 0x40, 0xc2, 0x06, 0x93, 0xde, 0xfa, 0x7c, 0xcc, 0x5b, 0xaf, 0x04, 0x45, 0xd2, 0x1c, 0xb1, - 0x47, 0x76, 0xd5, 0x8f, 0xa0, 0xd8, 0x96, 0x12, 0x3c, 0x7e, 0x27, 0x7d, 0x1a, 0x2a, 0x3c, 0x8d, - 0xb8, 0x9e, 0x31, 0x1a, 0x1f, 0x8f, 0x84, 0x97, 0x66, 0x68, 0xd9, 0xc7, 0xed, 0xb9, 0xfa, 0x26, - 0xe4, 0x3b, 0x06, 0x1f, 0x11, 0x12, 0xc4, 0xb3, 0x09, 0xe2, 0xe0, 0x14, 0x2d, 0x74, 0x8a, 0xfe, - 0xa9, 0x06, 0x10, 0xe8, 0xe2, 0x51, 0x6e, 0xb1, 0x0e, 0x05, 0x17, 0x85, 0x51, 0xed, 0xc0, 0x7c, - 0xa0, 0x3e, 0xc4, 0x4b, 0x7a, 0x45, 0x75, 0x69, 0x14, 0x92, 0x97, 0xc3, 0x16, 0xcf, 0xc6, 0x4a, - 0xb8, 0x52, 0xbc, 0xe4, 0x1a, 0x50, 0x3e, 0xf7, 0x3e, 0xcc, 0xc7, 0xa6, 0x0b, 0x52, 0x81, 0xe2, - 0xfe, 0xc1, 0x71, 0x9b, 0xd2, 0x03, 0x5a, 0x9b, 0x21, 0x57, 0x60, 0x7e, 0x6f, 0xf3, 0xbd, 0xe3, - 0xdd, 0x9d, 0xa3, 0xf6, 0x71, 0x97, 0x6e, 0xde, 0x69, 0x77, 0x6a, 0x1a, 0x47, 0xe2, 0xfa, 0xb8, - 0x7b, 0x70, 0x70, 0xbc, 0xbb, 0x49, 0xef, 0xb6, 0x6b, 0xb3, 0x64, 0x01, 0xaa, 0xef, 0xec, 0xbf, - 0xb5, 0x7f, 0xf0, 0xee, 0xbe, 0xdc, 0x9c, 0x69, 0xfd, 0x52, 0x83, 0x3c, 0x67, 0xcf, 0x1c, 0xf2, - 0x03, 0x28, 0xf9, 0x43, 0x0a, 0xb9, 0x1a, 0x19, 0x6d, 0xc2, 0x83, 0x4b, 0xe3, 0x89, 0xc8, 0x27, - 0xe5, 0x9c, 0xfa, 0x0c, 0xd9, 0x84, 0xb2, 0x4f, 0x7c, 0xd4, 0xfa, 0x3a, 0x2c, 0x5a, 0xff, 0xd6, - 0xa0, 0x26, 0xfd, 0xf2, 0x2e, 0xb3, 0x98, 0x63, 0x78, 0xb6, 0x2f, 0x18, 0xce, 0x2b, 0x31, 0xae, - 0xe1, 0xe1, 0xe7, 0x7c, 0xc1, 0x76, 0x00, 0xee, 0x32, 0x4f, 0xf5, 0x8a, 0xd7, 0xd2, 0x8b, 0xa3, - 0xe0, 0x71, 0xfd, 0x9c, 0xca, 0xa9, 0x58, 0xdd, 0x05, 0x08, 0x02, 0x93, 0x04, 0xb5, 0x3e, 0x91, - 0x5e, 0x1b, 0xd7, 0x52, 0xbf, 0xf9, 0x37, 0xfd, 0x5d, 0x16, 0x0a, 0xfc, 0x83, 0xc9, 0x1c, 0xf2, - 0x06, 0x54, 0x7f, 0x68, 0x5a, 0x7d, 0xff, 0xa7, 0x75, 0x72, 0x35, 0xed, 0x17, 0x7d, 0xc1, 0xb6, - 0x71, 0xfe, 0x8f, 0xfd, 0x68, 0x82, 0x8a, 0xfa, 0xb1, 0xae, 0xc7, 0x2c, 0x8f, 0x9c, 0xf3, 0x0b, - 0x71, 0xe3, 0xc9, 0x04, 0xde, 0x67, 0xd1, 0x86, 0x72, 0xe8, 0xd7, 0xe7, 0xb0, 0xb6, 0x12, 0xbf, - 0x49, 0x5f, 0xc4, 0xe6, 0x2e, 0x40, 0xf0, 0x66, 0x44, 0x2e, 0x78, 0xa5, 0x6e, 0x5c, 0x4b, 0xfd, - 0xe6, 0x33, 0x7a, 0x4b, 0x5d, 0x49, 0x3c, 0x3e, 0x5d, 0xc8, 0xea, 0xa9, 0xd4, 0x07, 0xb0, 0x10, - 0xb3, 0x23, 0x98, 0x8f, 0x3d, 0xbb, 0x90, 0xcb, 0x9e, 0x5a, 0x1b, 0xab, 0xe7, 0x13, 0xf8, 0x7c, - 0x7f, 0x14, 0x7a, 0x21, 0x53, 0xcf, 0x39, 0x97, 0x73, 0xd6, 0xcf, 0x23, 0x08, 0xcb, 0xdc, 0xfa, - 0x5b, 0x16, 0x6a, 0x1d, 0xcf, 0x61, 0xc6, 0xc8, 0xb4, 0x06, 0xca, 0x65, 0x5e, 0x83, 0xbc, 0x2c, - 0x7c, 0x0f, 0x6b, 0xe2, 0x0d, 0x8d, 0xc7, 0xc3, 0x63, 0xb1, 0xcd, 0x86, 0x46, 0xf6, 0x1e, 0xa3, - 0x75, 0x36, 0x34, 0xf2, 0xde, 0x37, 0x63, 0x9f, 0x0d, 0x8d, 0xbc, 0xff, 0xcd, 0x59, 0x68, 0x43, - 0x23, 0x87, 0xb0, 0x20, 0x73, 0xc5, 0x63, 0xc9, 0x0e, 0x1b, 0x1a, 0x39, 0x82, 0x2b, 0x61, 0x8e, - 0xb2, 0x85, 0x24, 0xd7, 0xa3, 0xfb, 0xa2, 0x4d, 0x72, 0x48, 0xc3, 0x69, 0xdd, 0x2e, 0xe7, 0xdb, - 0xfa, 0xa3, 0x06, 0x05, 0x95, 0x09, 0x8f, 0x53, 0xa7, 0x55, 0xfd, 0xa2, 0x19, 0x4e, 0x1e, 0xf4, - 0xcc, 0x85, 0x34, 0x8f, 0x3d, 0x5b, 0x6e, 0xd5, 0x3f, 0xfe, 0x62, 0x59, 0xfb, 0xf4, 0x8b, 0x65, - 0xed, 0x5f, 0x5f, 0x2c, 0x6b, 0xbf, 0xfa, 0x72, 0x79, 0xe6, 0xd3, 0x2f, 0x97, 0x67, 0x3e, 0xfb, - 0x72, 0x79, 0xe6, 0x24, 0x8f, 0xff, 0x3b, 0xf6, 0xd2, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x05, - 0xbf, 0xa3, 0xdb, 0xbc, 0x26, 0x00, 0x00, + 0xd1, 0x1a, 0xf1, 0xa9, 0x22, 0x29, 0x51, 0x2d, 0x59, 0xe6, 0x72, 0xd7, 0x92, 0x3c, 0x5e, 0x18, + 0xfa, 0xfc, 0xa0, 0xb4, 0xf4, 0x1a, 0x9f, 0xd7, 0x4e, 0x1c, 0x48, 0x2b, 0x66, 0x2d, 0x5b, 0x2f, + 0x37, 0x69, 0xd9, 0x08, 0x0c, 0x08, 0x23, 0xb2, 0x97, 0x3b, 0x10, 0x39, 0x43, 0xcf, 0x0c, 0x65, + 0x29, 0x07, 0x23, 0x09, 0x90, 0x43, 0x80, 0x1c, 0x72, 0x48, 0x0e, 0xf9, 0x05, 0x41, 0x72, 0xc9, + 0x21, 0xf9, 0x07, 0x41, 0x0c, 0x07, 0x41, 0x02, 0x03, 0xb9, 0x18, 0x39, 0x18, 0x81, 0x7d, 0x48, + 0x7e, 0x46, 0xd0, 0xd5, 0xdd, 0xf3, 0xa6, 0xe4, 0xb5, 0xd7, 0x88, 0x0f, 0x3e, 0xb1, 0xab, 0xba, + 0xba, 0xba, 0xba, 0xba, 0x9e, 0x3d, 0x84, 0xc7, 0x47, 0xa7, 0xfd, 0x75, 0x8f, 0x0d, 0x47, 0xf6, + 0xe8, 0x44, 0xfc, 0x36, 0x46, 0x8e, 0xed, 0xd9, 0xa4, 0x20, 0x91, 0xf5, 0xa5, 0xae, 0x3d, 0x1c, + 0xda, 0xd6, 0xfa, 0xd9, 0xad, 0x75, 0x31, 0x12, 0x04, 0xf5, 0xe7, 0xfb, 0xa6, 0xf7, 0x60, 0x7c, + 0xd2, 0xe8, 0xda, 0xc3, 0xf5, 0xbe, 0xdd, 0xb7, 0xd7, 0x11, 0x7d, 0x32, 0xbe, 0x8f, 0x10, 0x02, + 0x38, 0x92, 0xe4, 0x8b, 0x9e, 0x63, 0x74, 0x19, 0xe7, 0x82, 0x03, 0x81, 0xd5, 0xff, 0xa8, 0x41, + 0xb5, 0xc3, 0xe1, 0xad, 0x8b, 0x9d, 0x6d, 0xca, 0xde, 0x1b, 0x33, 0xd7, 0x23, 0x35, 0x28, 0x20, + 0xcd, 0xce, 0x76, 0x4d, 0x5b, 0xd5, 0xd6, 0xca, 0x54, 0x81, 0x64, 0x19, 0xe0, 0x64, 0x60, 0x77, + 0x4f, 0xdb, 0x9e, 0xe1, 0x78, 0xb5, 0xe9, 0x55, 0x6d, 0x6d, 0x86, 0x86, 0x30, 0xa4, 0x0e, 0x45, + 0x84, 0x5a, 0x56, 0xaf, 0x96, 0xc1, 0x59, 0x1f, 0x26, 0x37, 0x60, 0xe6, 0xbd, 0x31, 0x73, 0x2e, + 0xf6, 0xec, 0x1e, 0xab, 0xe5, 0x70, 0x32, 0x40, 0x90, 0xe7, 0x60, 0xde, 0x18, 0x0c, 0xec, 0xf7, + 0x0f, 0x0d, 0xc7, 0x33, 0x8d, 0x01, 0xca, 0x54, 0xcb, 0xaf, 0x6a, 0x6b, 0x45, 0x9a, 0x9c, 0xd0, + 0xff, 0xa3, 0xc1, 0x7c, 0x48, 0x6c, 0x77, 0x64, 0x5b, 0x2e, 0x23, 0x37, 0x21, 0x87, 0x82, 0xa2, + 0xd4, 0xa5, 0xe6, 0x6c, 0x43, 0xaa, 0xb0, 0x81, 0xa4, 0x54, 0x4c, 0x92, 0x17, 0xa0, 0x30, 0x64, + 0x9e, 0x63, 0x76, 0x5d, 0x3c, 0x40, 0xa9, 0x79, 0x2d, 0x4a, 0xc7, 0x59, 0xee, 0x09, 0x02, 0xaa, + 0x28, 0xc9, 0x1d, 0xc8, 0xbb, 0x9e, 0xe1, 0x8d, 0x5d, 0x3c, 0xd6, 0x6c, 0xf3, 0xc9, 0xe4, 0x1a, + 0x25, 0x46, 0xa3, 0x8d, 0x84, 0x54, 0x2e, 0xe0, 0xda, 0x1c, 0x32, 0xd7, 0x35, 0xfa, 0xac, 0x96, + 0xc5, 0x53, 0x2b, 0x50, 0x7f, 0x0a, 0xf2, 0x82, 0x96, 0x94, 0xa1, 0x78, 0xf7, 0x60, 0xef, 0x70, + 0xb7, 0xd5, 0x69, 0x55, 0xa7, 0x48, 0x09, 0x0a, 0x87, 0x9b, 0xb4, 0xb3, 0xb3, 0xb9, 0x5b, 0xd5, + 0x74, 0x12, 0xba, 0x20, 0x29, 0x96, 0xfe, 0xf7, 0x69, 0xa8, 0xb4, 0x99, 0xe1, 0x74, 0x1f, 0xa8, + 0x2b, 0x7b, 0x19, 0xb2, 0x1d, 0xa3, 0xef, 0xd6, 0xb4, 0xd5, 0xcc, 0x5a, 0xa9, 0xb9, 0xea, 0x4b, + 0x17, 0xa1, 0x6a, 0x70, 0x92, 0x96, 0xe5, 0x39, 0x17, 0x5b, 0xd9, 0x8f, 0x3e, 0x5d, 0x99, 0xa2, + 0xb8, 0x86, 0xdc, 0x84, 0xca, 0x9e, 0x69, 0x6d, 0x8f, 0x1d, 0xc3, 0x33, 0x6d, 0x6b, 0x4f, 0xa8, + 0xa5, 0x42, 0xa3, 0x48, 0xa4, 0x32, 0xce, 0x43, 0x54, 0x19, 0x49, 0x15, 0x46, 0x92, 0x45, 0xc8, + 0xed, 0x9a, 0x43, 0xd3, 0xc3, 0xa3, 0x56, 0xa8, 0x00, 0x38, 0xd6, 0x45, 0x8b, 0xc9, 0x09, 0x2c, + 0x02, 0xa4, 0x0a, 0x19, 0x66, 0xf5, 0xf0, 0x92, 0x2b, 0x94, 0x0f, 0x39, 0xdd, 0x9b, 0xdc, 0x22, + 0x6a, 0x45, 0x54, 0x94, 0x00, 0xc8, 0x1a, 0xcc, 0xb5, 0x47, 0x86, 0xe5, 0x1e, 0x32, 0x87, 0xff, + 0xb6, 0x99, 0x57, 0x9b, 0xc1, 0x35, 0x71, 0x74, 0xfd, 0xff, 0x61, 0xc6, 0x3f, 0x22, 0x67, 0x7f, + 0xca, 0x2e, 0xd0, 0x16, 0x66, 0x28, 0x1f, 0x72, 0xf6, 0x67, 0xc6, 0x60, 0xcc, 0xa4, 0xe1, 0x0a, + 0xe0, 0xe5, 0xe9, 0x97, 0x34, 0xfd, 0xc3, 0x0c, 0x10, 0xa1, 0xaa, 0x2d, 0x6e, 0xae, 0x4a, 0xab, + 0xb7, 0x61, 0xc6, 0x55, 0x0a, 0x94, 0x46, 0xb5, 0x94, 0xae, 0x5a, 0x1a, 0x10, 0xf2, 0x0b, 0x47, + 0xa3, 0xdf, 0xd9, 0x96, 0x1b, 0x29, 0x90, 0xbb, 0x00, 0x1e, 0xfd, 0x90, 0x1b, 0x83, 0xd0, 0x5f, + 0x80, 0xe0, 0x1a, 0x1e, 0x19, 0x7d, 0xe6, 0x76, 0x6c, 0xc1, 0x5a, 0xea, 0x30, 0x8a, 0xe4, 0x2e, + 0xc6, 0xac, 0xae, 0xdd, 0x33, 0xad, 0xbe, 0xf4, 0x22, 0x1f, 0xe6, 0x1c, 0x4c, 0xab, 0xc7, 0xce, + 0x39, 0xbb, 0xb6, 0xf9, 0x43, 0x26, 0x75, 0x1b, 0x45, 0x12, 0x1d, 0xca, 0x9e, 0xed, 0x19, 0x03, + 0xca, 0xba, 0xb6, 0xd3, 0x73, 0x6b, 0x05, 0x24, 0x8a, 0xe0, 0x38, 0x4d, 0xcf, 0xf0, 0x8c, 0x96, + 0xda, 0x49, 0x5c, 0x48, 0x04, 0xc7, 0xcf, 0x79, 0xc6, 0x1c, 0xd7, 0xb4, 0x2d, 0xbc, 0x8f, 0x19, + 0xaa, 0x40, 0x42, 0x20, 0xeb, 0xf2, 0xed, 0x61, 0x55, 0x5b, 0xcb, 0x52, 0x1c, 0xf3, 0xd0, 0x71, + 0xdf, 0xb6, 0x3d, 0xe6, 0xa0, 0x60, 0x25, 0xdc, 0x33, 0x84, 0x21, 0xdb, 0x50, 0xed, 0xb1, 0x9e, + 0xd9, 0x35, 0x3c, 0xd6, 0xbb, 0x6b, 0x0f, 0xc6, 0x43, 0xcb, 0xad, 0x95, 0xd1, 0x9a, 0x6b, 0xbe, + 0xca, 0xb7, 0xa3, 0x04, 0x34, 0xb1, 0x42, 0xff, 0x93, 0x06, 0x73, 0x31, 0x2a, 0x72, 0x1b, 0x72, + 0x6e, 0xd7, 0x1e, 0x31, 0xe9, 0xba, 0xcb, 0x93, 0xd8, 0x35, 0xda, 0x9c, 0x8a, 0x0a, 0x62, 0x7e, + 0x06, 0xcb, 0x18, 0x2a, 0x5b, 0xc1, 0x31, 0xb9, 0x05, 0x59, 0xef, 0x62, 0x24, 0xe2, 0xcb, 0x6c, + 0xf3, 0x89, 0x89, 0x8c, 0x3a, 0x17, 0x23, 0x46, 0x91, 0x54, 0x5f, 0x81, 0x1c, 0xb2, 0x25, 0x45, + 0xc8, 0xb6, 0x0f, 0x37, 0xf7, 0xab, 0x53, 0xdc, 0xd9, 0x69, 0xab, 0x7d, 0xf0, 0x16, 0xbd, 0xdb, + 0x42, 0xff, 0xce, 0x72, 0x72, 0x02, 0x90, 0x6f, 0x77, 0xe8, 0xce, 0xfe, 0xbd, 0xea, 0x94, 0x7e, + 0x0e, 0xb3, 0xca, 0xba, 0x64, 0x68, 0xbb, 0x0d, 0x79, 0x8c, 0x5e, 0xca, 0xc3, 0x6f, 0x44, 0xe3, + 0x8f, 0xa0, 0xde, 0x63, 0x9e, 0xc1, 0x6f, 0x88, 0x4a, 0x5a, 0xb2, 0x11, 0x0f, 0x75, 0x71, 0xeb, + 0x8d, 0xc7, 0x39, 0xfd, 0x1f, 0x19, 0x58, 0x48, 0xe1, 0x18, 0x4f, 0x09, 0x33, 0x41, 0x4a, 0x58, + 0x83, 0x39, 0xc7, 0xb6, 0xbd, 0x36, 0x73, 0xce, 0xcc, 0x2e, 0xdb, 0x0f, 0x54, 0x16, 0x47, 0x73, + 0xeb, 0xe4, 0x28, 0x64, 0x8f, 0x74, 0x22, 0x43, 0x44, 0x91, 0x3c, 0x11, 0xa0, 0x4b, 0x74, 0xcc, + 0x21, 0x7b, 0xcb, 0x32, 0xcf, 0xf7, 0x0d, 0xcb, 0x46, 0x4f, 0xc8, 0xd2, 0xe4, 0x04, 0xb7, 0xaa, + 0x5e, 0x10, 0x92, 0x44, 0x78, 0x09, 0x61, 0xc8, 0x33, 0x50, 0x70, 0x65, 0xcc, 0xc8, 0xa3, 0x06, + 0xaa, 0x81, 0x06, 0x04, 0x9e, 0x2a, 0x02, 0xf2, 0x1c, 0x14, 0xe5, 0x90, 0xfb, 0x44, 0x26, 0x95, + 0xd8, 0xa7, 0x20, 0x14, 0xca, 0xae, 0x38, 0x1c, 0x8f, 0xe1, 0x6e, 0xad, 0x88, 0x2b, 0x1a, 0x97, + 0xdd, 0x4b, 0xa3, 0x1d, 0x5a, 0x80, 0x41, 0x8a, 0x46, 0x78, 0xd4, 0x8f, 0x60, 0x3e, 0x41, 0x92, + 0x12, 0xc7, 0x9e, 0x0d, 0xc7, 0xb1, 0x52, 0xf3, 0xb1, 0xd0, 0xa5, 0x06, 0x8b, 0xc3, 0xe1, 0x6d, + 0x17, 0xca, 0xe1, 0x29, 0x8c, 0x43, 0x23, 0xc3, 0xba, 0x6b, 0x8f, 0x2d, 0x0f, 0x19, 0xf3, 0x38, + 0xa4, 0x10, 0x5c, 0xa7, 0xcc, 0x71, 0x6c, 0x47, 0x4c, 0x8b, 0x64, 0x10, 0xc2, 0xe8, 0x3f, 0xd5, + 0xa0, 0x20, 0xf5, 0x41, 0x9e, 0x82, 0x1c, 0x5f, 0xa8, 0xcc, 0xb2, 0x12, 0x51, 0x18, 0x15, 0x73, + 0x98, 0x01, 0x0d, 0xaf, 0xfb, 0x80, 0xf5, 0x24, 0x37, 0x05, 0x92, 0x57, 0x00, 0x0c, 0xcf, 0x73, + 0xcc, 0x93, 0xb1, 0xc7, 0x78, 0x46, 0xe1, 0x3c, 0xae, 0xfb, 0x3c, 0x64, 0xb9, 0x73, 0x76, 0xab, + 0xf1, 0x06, 0xbb, 0x38, 0xe2, 0xa7, 0xa1, 0x21, 0x72, 0xee, 0xeb, 0x59, 0xbe, 0x0d, 0x59, 0x82, + 0x3c, 0xdf, 0xc8, 0xb7, 0x4d, 0x09, 0xa5, 0xba, 0x70, 0xaa, 0x79, 0x65, 0x26, 0x99, 0xd7, 0x4d, + 0xa8, 0x28, 0x63, 0xe2, 0xb0, 0x2b, 0x0d, 0x31, 0x8a, 0x8c, 0x9d, 0x22, 0xf7, 0x70, 0xa7, 0xf8, + 0xb5, 0x9f, 0xcb, 0xa5, 0x33, 0x72, 0x8f, 0x32, 0x2d, 0x77, 0xc4, 0xba, 0x1e, 0xeb, 0x75, 0x94, + 0xd3, 0x63, 0xbe, 0x8b, 0xa1, 0xc9, 0xd3, 0x30, 0xeb, 0xa3, 0xb6, 0x2e, 0xf8, 0xe6, 0xd3, 0x28, + 0x5f, 0x0c, 0x4b, 0x56, 0xa1, 0x84, 0xd1, 0x1d, 0x93, 0x9b, 0xca, 0xdc, 0x61, 0x14, 0x3f, 0x68, + 0xd7, 0x1e, 0x8e, 0x06, 0xcc, 0x63, 0xbd, 0xd7, 0xed, 0x13, 0x57, 0xe5, 0x9e, 0x08, 0x92, 0xdb, + 0x0d, 0x2e, 0x42, 0x0a, 0xe1, 0x6c, 0x01, 0x82, 0xcb, 0x1d, 0xb0, 0x14, 0xe2, 0xe4, 0x51, 0x9c, + 0x38, 0x3a, 0x22, 0x37, 0xe6, 0x70, 0xcc, 0x41, 0x61, 0xb9, 0x11, 0xab, 0xff, 0x59, 0xe3, 0x0e, + 0xc1, 0x75, 0xc3, 0xd3, 0xba, 0xca, 0xca, 0x8b, 0x2a, 0x9e, 0x8b, 0xdb, 0x96, 0xf1, 0x7a, 0x11, + 0x72, 0x58, 0x4d, 0xaa, 0xe4, 0x8e, 0x40, 0x50, 0x79, 0x64, 0x52, 0x2a, 0x8f, 0x6c, 0x50, 0x79, + 0xac, 0xc1, 0xdc, 0xd0, 0x38, 0xe7, 0xbb, 0xf0, 0x72, 0x02, 0xb9, 0x8b, 0xf3, 0xc5, 0xd1, 0xa4, + 0x09, 0x8b, 0xae, 0x67, 0x0c, 0x18, 0xde, 0xa4, 0xdb, 0x79, 0xe0, 0x30, 0xf7, 0x81, 0x3d, 0x50, + 0x65, 0x4c, 0xea, 0x9c, 0xfe, 0xbb, 0x2c, 0x2c, 0x05, 0xe7, 0x88, 0x94, 0x18, 0x2f, 0x25, 0x4b, + 0x8c, 0x7a, 0x2c, 0x48, 0x87, 0xce, 0xfe, 0x6d, 0x99, 0xf1, 0x8d, 0x28, 0x33, 0xd2, 0xcc, 0xa5, + 0x92, 0x6e, 0x2e, 0x1b, 0xb0, 0x10, 0x98, 0x44, 0x60, 0x2d, 0xb3, 0x48, 0x9d, 0x36, 0xa5, 0x7f, + 0x92, 0x81, 0xeb, 0xfe, 0xc5, 0x0b, 0x4b, 0x8a, 0x58, 0xcc, 0x77, 0x93, 0x16, 0xb3, 0x92, 0xb4, + 0x18, 0xb1, 0xf0, 0x5b, 0xb3, 0xf9, 0x46, 0x55, 0xa7, 0x3d, 0xd5, 0x65, 0x08, 0x97, 0x96, 0xb5, + 0x5d, 0x1d, 0x8a, 0x9e, 0xd1, 0xe7, 0xc5, 0x8f, 0x48, 0xa3, 0x33, 0xd4, 0x87, 0x49, 0x33, 0x5e, + 0xc1, 0x05, 0xdb, 0xa9, 0xaa, 0x22, 0x51, 0xc3, 0x7d, 0x00, 0x8b, 0xc1, 0x2e, 0x47, 0x4d, 0x7f, + 0x9f, 0x26, 0xe4, 0x31, 0x54, 0xaa, 0x64, 0x9d, 0x16, 0x67, 0x8e, 0x9a, 0xa2, 0x08, 0x96, 0x94, + 0x5f, 0x6a, 0xff, 0x57, 0xc2, 0x41, 0x5b, 0x32, 0xf4, 0x73, 0xb1, 0x16, 0xca, 0xc5, 0x04, 0xb2, + 0x1e, 0x6f, 0x5a, 0xa7, 0xf1, 0xd0, 0x38, 0xd6, 0x3f, 0xd4, 0x42, 0xa1, 0x32, 0x62, 0xc4, 0x58, + 0x83, 0x0a, 0xbd, 0xf8, 0x35, 0xa8, 0x00, 0xaf, 0x8a, 0xfd, 0xd9, 0x94, 0xd8, 0x9f, 0x0b, 0x62, + 0xbf, 0x0e, 0x65, 0xe1, 0xb5, 0x62, 0x3b, 0x69, 0x96, 0x11, 0xdc, 0x24, 0x37, 0x2e, 0x4c, 0x76, + 0xe3, 0x53, 0x78, 0x3c, 0x71, 0x0e, 0x79, 0x11, 0x3c, 0x8d, 0xfa, 0xbb, 0x89, 0x1b, 0x0f, 0x10, + 0x5f, 0x4a, 0xe5, 0xb7, 0xa1, 0xa8, 0xb6, 0x41, 0xad, 0x5e, 0xf8, 0xd9, 0x11, 0xc7, 0xe9, 0x9d, + 0xaf, 0xfe, 0x23, 0x0d, 0xae, 0xc5, 0x64, 0x0c, 0x99, 0xcb, 0x7a, 0x5c, 0xca, 0x52, 0x73, 0x3e, + 0xa8, 0x6e, 0xe5, 0xcc, 0x57, 0x15, 0xfc, 0x2f, 0x1a, 0xcc, 0xc5, 0x26, 0x53, 0xaa, 0x1a, 0x2d, + 0xb5, 0xaa, 0x89, 0x54, 0x23, 0xd3, 0xf1, 0x6a, 0x24, 0x51, 0xd1, 0x64, 0xd2, 0x2a, 0x9a, 0x58, + 0x65, 0x94, 0x4d, 0x56, 0x46, 0x29, 0x55, 0x4d, 0x2e, 0xb5, 0xaa, 0xd1, 0xf7, 0x21, 0x87, 0x75, + 0x19, 0x69, 0x41, 0xc5, 0x61, 0xae, 0x3d, 0x76, 0xba, 0xac, 0x1d, 0x2a, 0x8e, 0x83, 0x28, 0x2d, + 0x5e, 0xe0, 0xce, 0x6e, 0x35, 0x68, 0x98, 0x8c, 0x46, 0x57, 0xe9, 0xfb, 0x50, 0x3e, 0x1c, 0xbb, + 0x41, 0x0f, 0xf8, 0x2a, 0x54, 0xb0, 0x0a, 0x77, 0xb7, 0x2e, 0x3a, 0xf2, 0x99, 0x2b, 0xb3, 0x36, + 0x1b, 0xd2, 0x32, 0xa7, 0x6e, 0x71, 0x0a, 0xca, 0x0c, 0xd7, 0xb6, 0x68, 0x94, 0x5c, 0x6f, 0x43, + 0x95, 0x53, 0xa0, 0xb0, 0xca, 0xa7, 0x9e, 0xf7, 0xfb, 0x4a, 0xee, 0x84, 0xe5, 0xad, 0xc7, 0x3e, + 0xfa, 0x74, 0x65, 0xea, 0x9f, 0x9f, 0xae, 0x54, 0x0e, 0x1d, 0x66, 0x0c, 0x06, 0x76, 0x57, 0x50, + 0xab, 0x86, 0xb2, 0x0a, 0x19, 0xb3, 0x27, 0x0a, 0xf5, 0x32, 0xe5, 0x43, 0x7d, 0x4f, 0x30, 0x15, + 0x07, 0x90, 0x4c, 0xef, 0x40, 0xe1, 0x04, 0x0b, 0xfc, 0x2f, 0x7c, 0x72, 0x45, 0xaf, 0xdf, 0x04, + 0x90, 0xaf, 0x5d, 0xfc, 0x86, 0x97, 0x22, 0x5d, 0x6f, 0x59, 0x89, 0xa1, 0xbf, 0x0a, 0x33, 0xbb, + 0xa6, 0x75, 0xda, 0x1e, 0x98, 0x5d, 0xde, 0x94, 0xe7, 0x06, 0xa6, 0x75, 0xaa, 0xf6, 0xba, 0x9e, + 0xdc, 0x8b, 0xef, 0xd1, 0xe0, 0x0b, 0xa8, 0xa0, 0xd4, 0x7f, 0xa2, 0x01, 0xe1, 0x48, 0x65, 0x8e, + 0x41, 0x61, 0x29, 0xc2, 0x88, 0x16, 0x0e, 0x23, 0x35, 0x28, 0xf4, 0x1d, 0x7b, 0x3c, 0xda, 0x52, + 0xe1, 0x45, 0x81, 0x9c, 0x7e, 0x80, 0x8f, 0x5d, 0xa2, 0x7f, 0x10, 0xc0, 0x17, 0x0d, 0x3b, 0xfa, + 0xcf, 0xb8, 0xf7, 0x05, 0x42, 0xb4, 0xc7, 0xc3, 0xa1, 0xe1, 0x5c, 0xfc, 0x6f, 0x64, 0xf9, 0xad, + 0x06, 0x0b, 0x11, 0x85, 0x04, 0x91, 0x8a, 0xb9, 0x9e, 0x39, 0xe4, 0x49, 0x0c, 0x25, 0x29, 0xd2, + 0x00, 0x11, 0x6d, 0x23, 0x45, 0xe7, 0x11, 0x6a, 0x23, 0x9f, 0x86, 0x59, 0xb4, 0xbf, 0xb6, 0x4f, + 0x22, 0x44, 0x8b, 0x61, 0x49, 0x23, 0x08, 0x1b, 0x59, 0xbc, 0xc1, 0xc5, 0x48, 0x13, 0x99, 0x08, + 0x19, 0xdf, 0x81, 0x32, 0x35, 0xde, 0x7f, 0xcd, 0x74, 0x3d, 0xbb, 0xef, 0x18, 0x43, 0x6e, 0x24, + 0x27, 0xe3, 0xee, 0x29, 0xf3, 0x64, 0x98, 0x90, 0x10, 0x3f, 0x7b, 0x37, 0x24, 0x99, 0x00, 0xf4, + 0xd7, 0xa1, 0xa8, 0xda, 0xb0, 0x94, 0xce, 0xfa, 0xb9, 0x68, 0x67, 0xbd, 0x14, 0xed, 0xe6, 0xdf, + 0xdc, 0xe5, 0xed, 0xb3, 0xd9, 0x55, 0xf1, 0xf3, 0x97, 0x1a, 0x94, 0x42, 0x22, 0x92, 0x2d, 0x98, + 0x1f, 0x18, 0x1e, 0xb3, 0xba, 0x17, 0xc7, 0x0f, 0x94, 0x78, 0xd2, 0x2a, 0x83, 0x1e, 0x3d, 0x2c, + 0x3b, 0xad, 0x4a, 0xfa, 0xe0, 0x34, 0xff, 0x07, 0x79, 0x97, 0x39, 0xa6, 0x74, 0xc8, 0x70, 0xc8, + 0xf5, 0xbb, 0x47, 0x49, 0xc0, 0x0f, 0x2e, 0x1c, 0x5c, 0x2a, 0x56, 0x42, 0xfa, 0xdf, 0xa2, 0xd6, + 0x2d, 0x0d, 0x2b, 0xd9, 0xf4, 0x5f, 0x71, 0x5b, 0xd3, 0xa9, 0xb7, 0x15, 0xc8, 0x97, 0xb9, 0x4a, + 0xbe, 0x2a, 0x64, 0x46, 0x77, 0xee, 0xc8, 0x96, 0x99, 0x0f, 0x05, 0xe6, 0x45, 0x19, 0x3f, 0xf9, + 0x50, 0x60, 0x36, 0x64, 0x9f, 0xc8, 0x87, 0x88, 0x79, 0x71, 0x43, 0x36, 0x84, 0x7c, 0xa8, 0xbf, + 0x0d, 0xf5, 0x34, 0x3f, 0x91, 0x26, 0x7a, 0x07, 0x66, 0x5c, 0x44, 0x99, 0x2c, 0x19, 0x02, 0x52, + 0xd6, 0x05, 0xd4, 0xfa, 0xaf, 0x34, 0xa8, 0x44, 0x2e, 0x36, 0x92, 0x3b, 0x73, 0x32, 0x77, 0x96, + 0x41, 0xb3, 0x50, 0x19, 0x19, 0xaa, 0x59, 0x1c, 0xba, 0x8f, 0xfa, 0xd6, 0xa8, 0x76, 0x9f, 0x43, + 0xae, 0x7c, 0xd5, 0xd7, 0x5c, 0x0e, 0x9d, 0xe0, 0xe1, 0x8a, 0x54, 0x3b, 0xe1, 0x50, 0x4f, 0x1e, + 0x4c, 0xeb, 0xe1, 0x1b, 0x85, 0xf8, 0x80, 0x50, 0x40, 0xde, 0xea, 0xeb, 0x00, 0x81, 0xec, 0xa9, + 0x69, 0xf5, 0xb0, 0x84, 0xcd, 0x51, 0x1c, 0xeb, 0x4c, 0x3c, 0x78, 0x4b, 0xc1, 0xb7, 0x0d, 0xcf, + 0xe0, 0xf5, 0xa9, 0xc3, 0xdc, 0xf1, 0xc0, 0xeb, 0x04, 0xa9, 0x3d, 0x84, 0xe1, 0xb5, 0x9d, 0x80, + 0xa4, 0xd9, 0xd4, 0x53, 0x7d, 0x08, 0x29, 0xa8, 0xa4, 0xe4, 0x51, 0x70, 0x3e, 0x31, 0xcb, 0xcd, + 0x64, 0x60, 0x9c, 0xb0, 0x41, 0xa8, 0xce, 0x0a, 0x10, 0x5c, 0x0e, 0x04, 0x8e, 0x42, 0xd5, 0x44, + 0x08, 0x43, 0xd6, 0x61, 0xda, 0x53, 0xa6, 0xb1, 0x32, 0x59, 0x86, 0x43, 0xdb, 0xb4, 0x3c, 0x3a, + 0xed, 0xb9, 0xdc, 0x87, 0x96, 0xd2, 0xa7, 0xf1, 0x32, 0x4c, 0x29, 0x44, 0x85, 0xe2, 0x98, 0x5b, + 0xc7, 0x99, 0x31, 0xc0, 0x8d, 0x35, 0xca, 0x87, 0x3c, 0x3f, 0xb3, 0x73, 0x36, 0x1c, 0x0d, 0x0c, + 0xa7, 0x23, 0x5f, 0x28, 0x33, 0xf8, 0xd1, 0x2a, 0x8e, 0x26, 0xcf, 0x40, 0x55, 0xa1, 0xd4, 0x17, + 0x0b, 0x69, 0x9c, 0x09, 0xbc, 0xde, 0x86, 0x05, 0xfc, 0xf8, 0xb0, 0x63, 0xb9, 0x9e, 0x61, 0x79, + 0x97, 0x47, 0x65, 0x3f, 0xca, 0xca, 0x48, 0x13, 0x89, 0xb2, 0xc2, 0x37, 0x31, 0xca, 0x9e, 0xc3, + 0x62, 0x94, 0xa9, 0x34, 0xe1, 0x86, 0xef, 0x53, 0xc2, 0x7e, 0x83, 0xb0, 0x23, 0x29, 0xdb, 0x38, + 0xeb, 0x3b, 0xd6, 0xc3, 0x3f, 0xeb, 0xfe, 0x58, 0x83, 0x4a, 0x84, 0x17, 0xb9, 0x03, 0x79, 0xbc, + 0xb6, 0xa4, 0xcf, 0x24, 0xdf, 0xab, 0xe4, 0xd7, 0x22, 0xb9, 0x20, 0x5a, 0x4c, 0x6a, 0x32, 0x18, + 0x92, 0x15, 0x28, 0x8d, 0x1c, 0x7b, 0x78, 0x2c, 0xb9, 0x8a, 0xb7, 0x5d, 0xe0, 0xa8, 0x5d, 0xc4, + 0xe8, 0xbf, 0xcf, 0xc0, 0x3c, 0x1e, 0x9f, 0x1a, 0x56, 0x9f, 0x3d, 0x12, 0x8d, 0x62, 0x2b, 0xe7, + 0xb1, 0x91, 0xbc, 0x46, 0x1c, 0x47, 0xbf, 0x33, 0x16, 0xe2, 0xdf, 0x19, 0x43, 0xed, 0x6f, 0xf1, + 0x92, 0xf6, 0x77, 0xe6, 0xca, 0xf6, 0x17, 0xd2, 0xda, 0xdf, 0x50, 0xd3, 0x59, 0x8a, 0x36, 0x9d, + 0xe1, 0xc6, 0xb8, 0x1c, 0x6b, 0x8c, 0x55, 0x43, 0x5a, 0x99, 0xd8, 0x90, 0xce, 0x7e, 0xa1, 0x86, + 0x74, 0xee, 0xa1, 0xdf, 0x31, 0x78, 0x7e, 0x97, 0xa6, 0xef, 0xd6, 0xaa, 0xe2, 0xcc, 0x3e, 0x42, + 0x77, 0x81, 0x84, 0x2f, 0x4c, 0x5a, 0xeb, 0xb3, 0x31, 0x6b, 0x5d, 0x08, 0x92, 0xa4, 0x39, 0x64, + 0x5f, 0xd9, 0x54, 0x3f, 0x80, 0x62, 0x4b, 0x4a, 0xf0, 0xe8, 0x8d, 0xf4, 0x49, 0x28, 0xf3, 0x30, + 0xe2, 0x7a, 0xc6, 0x70, 0x74, 0x3c, 0x14, 0x56, 0x9a, 0xa1, 0x25, 0x1f, 0xb7, 0xe7, 0xea, 0x9b, + 0x90, 0x6f, 0x1b, 0xbc, 0x45, 0x48, 0x10, 0x4f, 0x27, 0x88, 0x83, 0x5d, 0xb4, 0xd0, 0x2e, 0xfa, + 0xc7, 0x1a, 0x40, 0xa0, 0x8b, 0xaf, 0x72, 0x8a, 0x75, 0x28, 0xb8, 0x28, 0x8c, 0x2a, 0x07, 0xe6, + 0x02, 0xf5, 0x21, 0x5e, 0xd2, 0x2b, 0xaa, 0x2b, 0xbd, 0x90, 0xbc, 0x18, 0xbe, 0xf1, 0x6c, 0x2c, + 0x85, 0x2b, 0xc5, 0x4b, 0xae, 0x01, 0xe5, 0x33, 0xef, 0xc2, 0x5c, 0xac, 0xbb, 0x20, 0x65, 0x28, + 0xee, 0x1f, 0x1c, 0xb7, 0x28, 0x3d, 0xa0, 0xd5, 0x29, 0xb2, 0x00, 0x73, 0x7b, 0x9b, 0xef, 0x1c, + 0xef, 0xee, 0x1c, 0xb5, 0x8e, 0x3b, 0x74, 0xf3, 0x6e, 0xab, 0x5d, 0xd5, 0x38, 0x12, 0xc7, 0xc7, + 0x9d, 0x83, 0x83, 0xe3, 0xdd, 0x4d, 0x7a, 0xaf, 0x55, 0x9d, 0x26, 0xf3, 0x50, 0x79, 0x6b, 0xff, + 0x8d, 0xfd, 0x83, 0xb7, 0xf7, 0xe5, 0xe2, 0x4c, 0xf3, 0xe7, 0x1a, 0xe4, 0x39, 0x7b, 0xe6, 0x90, + 0xef, 0xc1, 0x8c, 0xdf, 0xa4, 0x90, 0x6b, 0x91, 0xd6, 0x26, 0xdc, 0xb8, 0xd4, 0x1f, 0x8b, 0x4c, + 0x29, 0xe3, 0xd4, 0xa7, 0xc8, 0x26, 0x94, 0x7c, 0xe2, 0xa3, 0xe6, 0x97, 0x61, 0xd1, 0xfc, 0xb7, + 0x06, 0x55, 0x69, 0x97, 0xf7, 0x98, 0xc5, 0x1c, 0xc3, 0xb3, 0x7d, 0xc1, 0xb0, 0x5f, 0x89, 0x71, + 0x0d, 0x37, 0x3f, 0x93, 0x05, 0xdb, 0x01, 0xb8, 0xc7, 0x3c, 0x55, 0x2b, 0x5e, 0x4f, 0x4f, 0x8e, + 0x82, 0xc7, 0x8d, 0x09, 0x99, 0x53, 0xb1, 0xba, 0x07, 0x10, 0x38, 0x26, 0x09, 0x72, 0x7d, 0x22, + 0xbc, 0xd6, 0xaf, 0xa7, 0xce, 0xf9, 0x27, 0xfd, 0x4d, 0x16, 0x0a, 0x7c, 0xc2, 0x64, 0x0e, 0x79, + 0x0d, 0x2a, 0xdf, 0x37, 0xad, 0x9e, 0xff, 0x67, 0x03, 0x72, 0x2d, 0xed, 0x3f, 0x0e, 0x82, 0x6d, + 0x7d, 0xf2, 0xdf, 0x1f, 0xf0, 0x0a, 0xca, 0xea, 0xf3, 0x65, 0x97, 0x59, 0x1e, 0x99, 0xf0, 0xcd, + 0xbc, 0xfe, 0x78, 0x02, 0xef, 0xb3, 0x68, 0x41, 0x29, 0xf4, 0x3d, 0x3e, 0xac, 0xad, 0xc4, 0x57, + 0xfa, 0xcb, 0xd8, 0xdc, 0x03, 0x08, 0x9e, 0xa2, 0xc8, 0x25, 0x0f, 0xeb, 0xf5, 0xeb, 0xa9, 0x73, + 0x3e, 0xa3, 0x37, 0xd4, 0x91, 0xc4, 0x9b, 0xd6, 0xa5, 0xac, 0x9e, 0x48, 0x7d, 0x57, 0x0b, 0x31, + 0x3b, 0x82, 0xb9, 0xd8, 0xb3, 0x0b, 0xb9, 0xea, 0x05, 0xb7, 0xbe, 0x3a, 0x99, 0xc0, 0xe7, 0xfb, + 0x83, 0xd0, 0xc3, 0x9b, 0x7a, 0xce, 0xb9, 0x9a, 0xb3, 0x3e, 0x89, 0x20, 0x2c, 0x73, 0xf3, 0xaf, + 0x59, 0xa8, 0xb6, 0x3d, 0x87, 0x19, 0x43, 0xd3, 0xea, 0x2b, 0x93, 0x79, 0x05, 0xf2, 0x32, 0xf1, + 0x3d, 0xec, 0x15, 0x6f, 0x68, 0xdc, 0x1f, 0x1e, 0xc9, 0xdd, 0x6c, 0x68, 0x64, 0xef, 0x11, 0xde, + 0xce, 0x86, 0x46, 0xde, 0xf9, 0x7a, 0xee, 0x67, 0x43, 0x23, 0xef, 0x7e, 0x7d, 0x37, 0xb4, 0xa1, + 0x91, 0x43, 0x98, 0x97, 0xb1, 0xe2, 0x91, 0x44, 0x87, 0x0d, 0x8d, 0x1c, 0xc1, 0x42, 0x98, 0xa3, + 0x2c, 0x21, 0xc9, 0x8d, 0xe8, 0xba, 0x68, 0x91, 0x1c, 0xd2, 0x70, 0x5a, 0xb5, 0xcb, 0xf9, 0x36, + 0xff, 0xa0, 0x41, 0x41, 0x45, 0xc2, 0xe3, 0xd4, 0x6e, 0x55, 0xbf, 0xac, 0x87, 0x93, 0x1b, 0x3d, + 0x75, 0x29, 0xcd, 0x23, 0x8f, 0x96, 0x5b, 0xb5, 0x8f, 0x3e, 0x5b, 0xd6, 0x3e, 0xfe, 0x6c, 0x59, + 0xfb, 0xd7, 0x67, 0xcb, 0xda, 0x2f, 0x3e, 0x5f, 0x9e, 0xfa, 0xf8, 0xf3, 0xe5, 0xa9, 0x4f, 0x3e, + 0x5f, 0x9e, 0x3a, 0xc9, 0xe3, 0xbf, 0xe9, 0x5e, 0xf8, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3c, + 0x55, 0xe8, 0xef, 0xce, 0x27, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -5610,6 +5663,16 @@ func (m *SearchTagsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.StaleValuesThreshold != 0 { + i = encodeVarintTempo(dAtA, i, uint64(m.StaleValuesThreshold)) + i-- + dAtA[i] = 0x30 + } + if m.MaxTagsPerScope != 0 { + i = encodeVarintTempo(dAtA, i, uint64(m.MaxTagsPerScope)) + i-- + dAtA[i] = 0x28 + } if m.End != 0 { i = encodeVarintTempo(dAtA, i, uint64(m.End)) i-- @@ -5657,6 +5720,16 @@ func (m *SearchTagsBlockRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) _ = i var l int _ = l + if m.StaleValueThreshold != 0 { + i = encodeVarintTempo(dAtA, i, uint64(m.StaleValueThreshold)) + i-- + dAtA[i] = 0x70 + } + if m.MaxTagsPerScope != 0 { + i = encodeVarintTempo(dAtA, i, uint64(m.MaxTagsPerScope)) + i-- + dAtA[i] = 0x68 + } if len(m.DedicatedColumns) > 0 { for iNdEx := len(m.DedicatedColumns) - 1; iNdEx >= 0; iNdEx-- { { @@ -6003,6 +6076,16 @@ func (m *SearchTagValuesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) _ = i var l int _ = l + if m.StaleValueThreshold != 0 { + i = encodeVarintTempo(dAtA, i, uint64(m.StaleValueThreshold)) + i-- + dAtA[i] = 0x38 + } + if m.MaxTagValues != 0 { + i = encodeVarintTempo(dAtA, i, uint64(m.MaxTagValues)) + i-- + dAtA[i] = 0x30 + } if m.End != 0 { i = encodeVarintTempo(dAtA, i, uint64(m.End)) i-- @@ -7872,6 +7955,12 @@ func (m *SearchTagsRequest) Size() (n int) { if m.End != 0 { n += 1 + sovTempo(uint64(m.End)) } + if m.MaxTagsPerScope != 0 { + n += 1 + sovTempo(uint64(m.MaxTagsPerScope)) + } + if m.StaleValuesThreshold != 0 { + n += 1 + sovTempo(uint64(m.StaleValuesThreshold)) + } return n } @@ -7925,6 +8014,12 @@ func (m *SearchTagsBlockRequest) Size() (n int) { n += 1 + l + sovTempo(uint64(l)) } } + if m.MaxTagsPerScope != 0 { + n += 1 + sovTempo(uint64(m.MaxTagsPerScope)) + } + if m.StaleValueThreshold != 0 { + n += 1 + sovTempo(uint64(m.StaleValueThreshold)) + } return n } @@ -8058,6 +8153,12 @@ func (m *SearchTagValuesRequest) Size() (n int) { if m.End != 0 { n += 1 + sovTempo(uint64(m.End)) } + if m.MaxTagValues != 0 { + n += 1 + sovTempo(uint64(m.MaxTagValues)) + } + if m.StaleValueThreshold != 0 { + n += 1 + sovTempo(uint64(m.StaleValueThreshold)) + } return n } @@ -11182,6 +11283,44 @@ func (m *SearchTagsRequest) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxTagsPerScope", wireType) + } + m.MaxTagsPerScope = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTempo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxTagsPerScope |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StaleValuesThreshold", wireType) + } + m.StaleValuesThreshold = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTempo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StaleValuesThreshold |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipTempo(dAtA[iNdEx:]) @@ -11544,6 +11683,44 @@ func (m *SearchTagsBlockRequest) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 13: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxTagsPerScope", wireType) + } + m.MaxTagsPerScope = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTempo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxTagsPerScope |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 14: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StaleValueThreshold", wireType) + } + m.StaleValueThreshold = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTempo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StaleValueThreshold |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipTempo(dAtA[iNdEx:]) @@ -12410,6 +12587,44 @@ func (m *SearchTagValuesRequest) Unmarshal(dAtA []byte) error { break } } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxTagValues", wireType) + } + m.MaxTagValues = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTempo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxTagValues |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StaleValueThreshold", wireType) + } + m.StaleValueThreshold = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTempo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StaleValueThreshold |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipTempo(dAtA[iNdEx:]) diff --git a/pkg/tempopb/tempo.proto b/pkg/tempopb/tempo.proto index 53dba1d68e4..6b0939f64b1 100644 --- a/pkg/tempopb/tempo.proto +++ b/pkg/tempopb/tempo.proto @@ -163,6 +163,8 @@ message SearchTagsRequest { string query = 2; uint32 start = 3; uint32 end = 4; + uint32 maxTagsPerScope = 5; + uint32 staleValuesThreshold =6; } // SearchTagsBlockRequest takes SearchTagsRequest parameters as well as all information necessary @@ -180,6 +182,8 @@ message SearchTagsBlockRequest { uint64 size = 10; // total size of data file uint32 footerSize = 11; // size of file footer (parquet) repeated DedicatedColumn dedicatedColumns = 12; + uint32 maxTagsPerScope = 13; // Limit of tags per scope + uint32 staleValueThreshold = 14; // Limit of stale values } message SearchTagValuesBlockRequest { @@ -218,6 +222,8 @@ message SearchTagValuesRequest { string query = 2; // TraceQL query uint32 start = 4; uint32 end = 5; + uint32 maxTagValues = 6; + uint32 staleValueThreshold = 7; // Limit of stale values } message SearchTagValuesResponse { diff --git a/pkg/traceql/engine_test.go b/pkg/traceql/engine_test.go index f7f0599b4e4..a578d2451e0 100644 --- a/pkg/traceql/engine_test.go +++ b/pkg/traceql/engine_test.go @@ -668,7 +668,7 @@ func TestExecuteTagValues(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - distinctValues := collector.NewDistinctValue[tempopb.TagValue](100_000, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + distinctValues := collector.NewDistinctValue[tempopb.TagValue](100_000, 0, 0, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) // Ugly hack to make the mock fetcher work with a bad query fetcherQuery := tc.query diff --git a/tempodb/compactor_test.go b/tempodb/compactor_test.go index 3ef1f679eb1..0cf76ee4230 100644 --- a/tempodb/compactor_test.go +++ b/tempodb/compactor_test.go @@ -630,11 +630,11 @@ func testCompactionHonorsBlockStartEndTimes(t *testing.T, targetBlockVersion str r.EnablePolling(ctx, &mockJobSharder{}) - cutTestBlockWithTraces(t, w, testTenantID, []testData{ + cutTestBlockWithTraces(t, w, []testData{ {test.ValidTraceID(nil), test.MakeTrace(10, nil), 100, 101}, {test.ValidTraceID(nil), test.MakeTrace(10, nil), 102, 103}, }) - cutTestBlockWithTraces(t, w, testTenantID, []testData{ + cutTestBlockWithTraces(t, w, []testData{ {test.ValidTraceID(nil), test.MakeTrace(10, nil), 104, 105}, {test.ValidTraceID(nil), test.MakeTrace(10, nil), 106, 107}, }) @@ -784,7 +784,7 @@ type testData struct { start, end uint32 } -func cutTestBlockWithTraces(t testing.TB, w Writer, tenantID string, data []testData) common.BackendBlock { +func cutTestBlockWithTraces(t testing.TB, w Writer, data []testData) common.BackendBlock { dec := model.MustNewSegmentDecoder(model.CurrentEncoding) wal := w.WAL() diff --git a/tempodb/encoding/vparquet2/block_search_tags_test.go b/tempodb/encoding/vparquet2/block_search_tags_test.go index a32e1bcb0f5..58afd30e787 100644 --- a/tempodb/encoding/vparquet2/block_search_tags_test.go +++ b/tempodb/encoding/vparquet2/block_search_tags_test.go @@ -177,7 +177,7 @@ func BenchmarkBackendBlockSearchTags(b *testing.B) { block := newBackendBlock(meta, rr) opts := common.DefaultSearchOptions() - d := collector.NewDistinctString(1_000_000) + d := collector.NewDistinctString(1_000_000, 0, 0) mc := collector.NewMetricsCollector() b.ResetTimer() @@ -212,7 +212,7 @@ func BenchmarkBackendBlockSearchTagValues(b *testing.B) { for _, tc := range testCases { b.Run(tc, func(b *testing.B) { - d := collector.NewDistinctString(1_000_000) + d := collector.NewDistinctString(1_000_000, 0, 0) mc := collector.NewMetricsCollector() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index b4964c45f11..c95a8e1238a 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -211,7 +211,7 @@ func TestFetchTagNames(t *testing.T) { } t.Run(fmt.Sprintf("query: %s %s-%s", tc.name, tc.query, scope), func(t *testing.T) { - distinctAttrNames := collector.NewScopedDistinctString(0) + distinctAttrNames := collector.NewScopedDistinctString(0, 0, 0) req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(t, err) @@ -494,7 +494,7 @@ func TestFetchTagValues(t *testing.T) { for _, tc := range testCases { t.Run(fmt.Sprintf("tag: %s, query: %s", tc.tag, tc.query), func(t *testing.T) { - distinctValues := collector.NewDistinctValue[tempopb.TagValue](1_000_000, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + distinctValues := collector.NewDistinctValue[tempopb.TagValue](1_000_000, 0, 0, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(t, err) @@ -596,7 +596,7 @@ func BenchmarkFetchTagValues(b *testing.B) { for _, tc := range testCases { b.Run(fmt.Sprintf("tag: %s, query: %s", tc.tag, tc.query), func(b *testing.B) { - distinctValues := collector.NewDistinctValue[tempopb.TagValue](1_000_000, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + distinctValues := collector.NewDistinctValue[tempopb.TagValue](1_000_000, 0, 0, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(b, err) @@ -678,7 +678,7 @@ func BenchmarkFetchTags(b *testing.B) { for _, tc := range testCases { for _, scope := range []traceql.AttributeScope{traceql.AttributeScopeSpan, traceql.AttributeScopeResource, traceql.AttributeScopeNone} { b.Run(fmt.Sprintf("query: %s %s", tc.query, scope), func(b *testing.B) { - distinctStrings := collector.NewScopedDistinctString(1_000_000) + distinctStrings := collector.NewScopedDistinctString(1_000_000, 0, 0) req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(b, err) diff --git a/tempodb/encoding/vparquet3/block_search_tags_test.go b/tempodb/encoding/vparquet3/block_search_tags_test.go index 31e1ae2a704..c47a5c19bc8 100644 --- a/tempodb/encoding/vparquet3/block_search_tags_test.go +++ b/tempodb/encoding/vparquet3/block_search_tags_test.go @@ -203,7 +203,7 @@ func BenchmarkBackendBlockSearchTags(b *testing.B) { block := newBackendBlock(meta, rr) opts := common.DefaultSearchOptions() - d := collector.NewDistinctString(1_000_000) + d := collector.NewDistinctString(1_000_000, 0, 0) mc := collector.NewMetricsCollector() b.ResetTimer() @@ -238,7 +238,7 @@ func BenchmarkBackendBlockSearchTagValues(b *testing.B) { for _, tc := range testCases { b.Run(tc, func(b *testing.B) { - d := collector.NewDistinctString(1_000_000) + d := collector.NewDistinctString(1_000_000, 0, 0) mc := collector.NewMetricsCollector() b.ResetTimer() diff --git a/tempodb/encoding/vparquet4/block_autocomplete_test.go b/tempodb/encoding/vparquet4/block_autocomplete_test.go index d0f87aa82af..94166263222 100644 --- a/tempodb/encoding/vparquet4/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet4/block_autocomplete_test.go @@ -318,7 +318,7 @@ func TestFetchTagNames(t *testing.T) { } t.Run(fmt.Sprintf("query: %s %s-%s", tc.name, tc.query, scope), func(t *testing.T) { - distinctAttrNames := collector.NewScopedDistinctString(0) + distinctAttrNames := collector.NewScopedDistinctString(0, 0, 0) req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(t, err) @@ -617,7 +617,7 @@ func TestFetchTagValues(t *testing.T) { for _, tc := range testCases { t.Run(fmt.Sprintf("tag: %s, query: %s", tc.tag, tc.query), func(t *testing.T) { - distinctValues := collector.NewDistinctValue[tempopb.TagValue](1_000_000, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + distinctValues := collector.NewDistinctValue[tempopb.TagValue](1_000_000, 0, 0, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(t, err) @@ -720,7 +720,7 @@ func BenchmarkFetchTagValues(b *testing.B) { for _, tc := range testCases { b.Run(fmt.Sprintf("tag: %s, query: %s", tc.tag, tc.query), func(b *testing.B) { - distinctValues := collector.NewDistinctValue[tempopb.TagValue](1_000_000, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + distinctValues := collector.NewDistinctValue[tempopb.TagValue](1_000_000, 0, 0, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(b, err) @@ -802,7 +802,7 @@ func BenchmarkFetchTags(b *testing.B) { for _, tc := range testCases { for _, scope := range []traceql.AttributeScope{traceql.AttributeScopeSpan, traceql.AttributeScopeResource, traceql.AttributeScopeNone} { b.Run(fmt.Sprintf("query: %s %s", tc.query, scope), func(b *testing.B) { - distinctStrings := collector.NewScopedDistinctString(1_000_000) + distinctStrings := collector.NewScopedDistinctString(1_000_000, 0, 0) req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(b, err) diff --git a/tempodb/encoding/vparquet4/block_search_tags_test.go b/tempodb/encoding/vparquet4/block_search_tags_test.go index 2c734da42b1..d502d7bf7ae 100644 --- a/tempodb/encoding/vparquet4/block_search_tags_test.go +++ b/tempodb/encoding/vparquet4/block_search_tags_test.go @@ -203,7 +203,7 @@ func BenchmarkBackendBlockSearchTags(b *testing.B) { block := newBackendBlock(meta, rr) opts := common.DefaultSearchOptions() - d := collector.NewDistinctString(1_000_000) + d := collector.NewDistinctString(1_000_000, 0, 0) mc := collector.NewMetricsCollector() b.ResetTimer() @@ -238,7 +238,7 @@ func BenchmarkBackendBlockSearchTagValues(b *testing.B) { for _, tc := range testCases { b.Run(tc, func(b *testing.B) { - d := collector.NewDistinctString(1_000_000) + d := collector.NewDistinctString(1_000_000, 0, 0) mc := collector.NewMetricsCollector() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/tempodb/tempodb.go b/tempodb/tempodb.go index 86ceebed9bc..f8c46b16802 100644 --- a/tempodb/tempodb.go +++ b/tempodb/tempodb.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/grafana/dskit/user" "github.com/grafana/tempo/modules/cache/memcached" "github.com/grafana/tempo/modules/cache/redis" "github.com/grafana/tempo/pkg/cache" @@ -82,8 +83,8 @@ type IterateObjectCallback func(id common.ID, obj []byte) bool type Reader interface { Find(ctx context.Context, tenantID string, id common.ID, blockStart string, blockEnd string, timeStart int64, timeEnd int64, opts common.SearchOptions) ([]*tempopb.Trace, []error, error) Search(ctx context.Context, meta *backend.BlockMeta, req *tempopb.SearchRequest, opts common.SearchOptions) (*tempopb.SearchResponse, error) - SearchTags(ctx context.Context, meta *backend.BlockMeta, scope string, opts common.SearchOptions) (*tempopb.SearchTagsV2Response, error) - SearchTagValues(ctx context.Context, meta *backend.BlockMeta, tag string, opts common.SearchOptions) (*tempopb.SearchTagValuesResponse, error) + SearchTags(ctx context.Context, meta *backend.BlockMeta, req *tempopb.SearchTagsBlockRequest, opts common.SearchOptions) (*tempopb.SearchTagsV2Response, error) + SearchTagValues(ctx context.Context, meta *backend.BlockMeta, req *tempopb.SearchTagValuesBlockRequest, opts common.SearchOptions) (*tempopb.SearchTagValuesResponse, error) SearchTagValuesV2(ctx context.Context, meta *backend.BlockMeta, req *tempopb.SearchTagValuesRequest, opts common.SearchOptions) (*tempopb.SearchTagValuesV2Response, error) // TODO(suraj): use common.MetricsCallback in Fetch and remove the Bytes callback from traceql.FetchSpansResponse @@ -369,7 +370,8 @@ func (rw *readerWriter) Search(ctx context.Context, meta *backend.BlockMeta, req return block.Search(ctx, req, opts) } -func (rw *readerWriter) SearchTags(ctx context.Context, meta *backend.BlockMeta, scope string, opts common.SearchOptions) (*tempopb.SearchTagsV2Response, error) { +func (rw *readerWriter) SearchTags(ctx context.Context, meta *backend.BlockMeta, req *tempopb.SearchTagsBlockRequest, opts common.SearchOptions) (*tempopb.SearchTagsV2Response, error) { + scope := req.SearchReq.Scope attributeScope := traceql.AttributeScopeFromString(scope) if attributeScope == traceql.AttributeScopeUnknown { @@ -381,7 +383,7 @@ func (rw *readerWriter) SearchTags(ctx context.Context, meta *backend.BlockMeta, return nil, err } - distinctValues := collector.NewScopedDistinctString(0) // todo: propagate limit? + distinctValues := collector.NewScopedDistinctString(0, req.SearchReq.MaxTagsPerScope, req.SearchReq.StaleValuesThreshold) mc := collector.NewMetricsCollector() rw.cfg.Search.ApplyToOptions(&opts) @@ -392,6 +394,11 @@ func (rw *readerWriter) SearchTags(ctx context.Context, meta *backend.BlockMeta, return nil, err } + orgID, _ := user.ExtractOrgID(ctx) + if distinctValues.Exceeded() { + level.Warn(log.Logger).Log("msg", "Search tags exceeded limit, reduce cardinality or size of tags", "orgID", orgID, "stopReason", distinctValues.StopReason()) + } + // build response collected := distinctValues.Strings() resp := &tempopb.SearchTagsV2Response{ @@ -408,16 +415,21 @@ func (rw *readerWriter) SearchTags(ctx context.Context, meta *backend.BlockMeta, return resp, nil } -func (rw *readerWriter) SearchTagValues(ctx context.Context, meta *backend.BlockMeta, tag string, opts common.SearchOptions) (response *tempopb.SearchTagValuesResponse, err error) { +func (rw *readerWriter) SearchTagValues(ctx context.Context, meta *backend.BlockMeta, req *tempopb.SearchTagValuesBlockRequest, opts common.SearchOptions) (response *tempopb.SearchTagValuesResponse, err error) { block, err := encoding.OpenBlock(meta, rw.r) if err != nil { return &tempopb.SearchTagValuesResponse{}, err } - dv := collector.NewDistinctString(0) + dv := collector.NewDistinctString(0, req.SearchReq.MaxTagValues, req.SearchReq.StaleValueThreshold) mc := collector.NewMetricsCollector() rw.cfg.Search.ApplyToOptions(&opts) - err = block.SearchTagValues(ctx, tag, dv.Collect, mc.Add, opts) + err = block.SearchTagValues(ctx, req.SearchReq.TagName, dv.Collect, mc.Add, opts) + + orgID, _ := user.ExtractOrgID(ctx) + if dv.Exceeded() { + level.Warn(log.Logger).Log("msg", "Search tags exceeded limit, reduce cardinality or size of tags", "orgID", orgID, "stopReason", dv.StopReason()) + } return &tempopb.SearchTagValuesResponse{ TagValues: dv.Strings(), @@ -436,7 +448,7 @@ func (rw *readerWriter) SearchTagValuesV2(ctx context.Context, meta *backend.Blo return nil, err } - dv := collector.NewDistinctValue[tempopb.TagValue](0, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) + dv := collector.NewDistinctValue(0, req.MaxTagValues, req.StaleValueThreshold, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) }) mc := collector.NewMetricsCollector() rw.cfg.Search.ApplyToOptions(&opts) err = block.SearchTagValuesV2(ctx, tag, traceql.MakeCollectTagValueFunc(dv.Collect), mc.Add, opts) @@ -444,6 +456,11 @@ func (rw *readerWriter) SearchTagValuesV2(ctx context.Context, meta *backend.Blo return nil, err } + orgID, _ := user.ExtractOrgID(ctx) + if dv.Exceeded() { + level.Warn(log.Logger).Log("msg", "Search tags exceeded limit, reduce cardinality or size of tags", "orgID", orgID, "stopReason", dv.StopReason()) + } + resp := &tempopb.SearchTagValuesV2Response{ Metrics: &tempopb.MetadataMetrics{InspectedBytes: mc.TotalValue()}, } diff --git a/tempodb/tempodb_search_test.go b/tempodb/tempodb_search_test.go index 58d72e8ba08..e3f57642621 100644 --- a/tempodb/tempodb_search_test.go +++ b/tempodb/tempodb_search_test.go @@ -1383,7 +1383,7 @@ func tagValuesRunner(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetad for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { - valueCollector := collector.NewDistinctValue[tempopb.TagValue](0, func(_ tempopb.TagValue) int { return 0 }) + valueCollector := collector.NewDistinctValue[tempopb.TagValue](0, 0, 0, func(_ tempopb.TagValue) int { return 0 }) mc := collector.NewMetricsCollector() fetcher := traceql.NewTagValuesFetcherWrapper(func(ctx context.Context, req traceql.FetchTagValuesRequest, cb traceql.FetchTagValuesCallback) error { return bb.FetchTagValues(ctx, req, cb, mc.Add, common.DefaultSearchOptions()) @@ -1459,7 +1459,7 @@ func tagNamesRunner(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetada return bb.FetchTagNames(ctx, req, cb, mc.Add, common.DefaultSearchOptions()) }) - valueCollector := collector.NewScopedDistinctString(0) + valueCollector := collector.NewScopedDistinctString(0, 0, 0) err := e.ExecuteTagNames(ctx, traceql.AttributeScopeFromString(tc.scope), tc.query, func(tag string, scope traceql.AttributeScope) bool { return valueCollector.Collect(scope.String(), tag) }, fetcher) @@ -2293,7 +2293,13 @@ func TestSearchForTagsAndTagValues(t *testing.T) { block, err := w.CompleteBlock(context.Background(), head) require.NoError(t, err) - resp, err := r.SearchTags(context.Background(), block.BlockMeta(), "", common.DefaultSearchOptions()) + reqBlocks := &tempopb.SearchTagsBlockRequest{ + SearchReq: &tempopb.SearchTagsRequest{ + Scope: "", + }, + } + + resp, err := r.SearchTags(context.Background(), block.BlockMeta(), reqBlocks, common.DefaultSearchOptions()) require.NoError(t, err) expectedTags := []string{"stringTag", "intTag", "service.name", "other"} var actualTags []string @@ -2304,7 +2310,13 @@ func TestSearchForTagsAndTagValues(t *testing.T) { sort.Strings(actualTags) assert.Equal(t, expectedTags, actualTags) - respValues, err := r.SearchTagValues(context.Background(), block.BlockMeta(), "service.name", common.DefaultSearchOptions()) + req := &tempopb.SearchTagValuesBlockRequest{ + SearchReq: &tempopb.SearchTagValuesRequest{ + TagName: "service.name", + }, + } + + respValues, err := r.SearchTagValues(context.Background(), block.BlockMeta(), req, common.DefaultSearchOptions()) require.NotZero(t, respValues.Metrics.InspectedBytes) require.NoError(t, err) @@ -2313,7 +2325,13 @@ func TestSearchForTagsAndTagValues(t *testing.T) { sort.Strings(respValues.TagValues) assert.Equal(t, expectedTagsValues, respValues.TagValues) - respValues, err = r.SearchTagValues(context.Background(), block.BlockMeta(), "intTag", common.DefaultSearchOptions()) + req = &tempopb.SearchTagValuesBlockRequest{ + SearchReq: &tempopb.SearchTagValuesRequest{ + TagName: "intTag", + }, + } + + respValues, err = r.SearchTagValues(context.Background(), block.BlockMeta(), req, common.DefaultSearchOptions()) require.NotZero(t, respValues.Metrics.InspectedBytes) require.NoError(t, err) @@ -2372,7 +2390,7 @@ func TestSearchForTagsAndTagValues(t *testing.T) { require.Equal(t, expected, tagValues.TagValues) require.NotZero(t, tagValues.Metrics.InspectedBytes) - valueCollector := collector.NewDistinctValue[tempopb.TagValue](0, func(_ tempopb.TagValue) int { return 0 }) + valueCollector := collector.NewDistinctValue[tempopb.TagValue](0, 0, 0, func(_ tempopb.TagValue) int { return 0 }) mc := collector.NewMetricsCollector() f := traceql.NewTagValuesFetcherWrapper(func(ctx context.Context, req traceql.FetchTagValuesRequest, cb traceql.FetchTagValuesCallback) error { return r.FetchTagValues(ctx, block.BlockMeta(), req, cb, mc.Add, common.DefaultSearchOptions())