Skip to content

Commit

Permalink
autocomplete
Browse files Browse the repository at this point in the history
  • Loading branch information
@ie-pham
ie-pham committed Aug 14, 2024
1 parent 634b60f commit 581fed2
Show file tree
Hide file tree
Showing 10 changed files with 210 additions and 27 deletions.
2 changes: 1 addition & 1 deletion integration/e2e/multi_tenant_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ func testSearch(t *testing.T, tenant string, tenantSize int) {

tagsV2Resp, err := apiClient.SearchTagsV2()
require.NoError(t, err)
require.Equal(t, 4, len(tagsV2Resp.GetScopes())) // resource, span, event, intrinsics
require.Equal(t, 4, len(tagsV2Resp.GetScopes())) // resource, span, event, link, scope intrinsics
for _, s := range tagsV2Resp.Scopes {
require.NotEmpty(t, s.Tags)
}
Expand Down
26 changes: 26 additions & 0 deletions modules/ingester/instance_search_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,26 @@ func testSearchTagsAndValuesV2(
sort.Strings(expectedLinkTagValues)
assert.Contains(t, tagsResp.TagNames, tagName)
assert.Equal(t, expectedLinkTagValues, tagValues)

// scope scope attr

tagValuesResp, err = i.SearchTagValuesV2(ctx, &tempopb.SearchTagValuesRequest{
TagName: fmt.Sprintf("scope.%s", tagName),
Query: query,
})
require.NoError(t, err)

tagValues = make([]string, 0, len(tagValuesResp.TagValues))
for _, v := range tagValuesResp.TagValues {
tagValues = append(tagValues, v.Value)
}

sort.Strings(tagValues)
sort.Strings(expectedTagValues)
assert.Contains(t, tagsResp.TagNames, tagName)
assert.Equal(t, expectedTagValues, tagValues)


}

// TestInstanceSearchTagsSpecialCases tess that SearchTags errors on an unknown scope and
Expand All @@ -425,6 +445,7 @@ func TestInstanceSearchTagsSpecialCases(t *testing.T) {
t,
[]string{
"duration", "event:name", "event:timeSinceStart", "kind", "name", "rootName", "rootServiceName",
"scope:name", "scope:version",
"span:duration", "span:kind", "span:name", "span:status", "span:statusMessage", "status", "statusMessage",
"trace:duration", "trace:rootName", "trace:rootService", "traceDuration",
},
Expand Down Expand Up @@ -562,6 +583,11 @@ func writeTracesForSearch(t *testing.T, i *instance, spanName, tagKey, tagValue
// add the time
for _, batch := range testTrace.ResourceSpans {
for _, ils := range batch.ScopeSpans {
ils.Scope = &v1.InstrumentationScope{
Name: "scope-name",
Version: "scope-version",
Attributes: []*v1.KeyValue{kv},
}
for _, span := range ils.Spans {
span.Name = spanName
span.StartTimeUnixNano = uint64(now.UnixNano())
Expand Down
2 changes: 2 additions & 0 deletions pkg/search/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ func GetVirtualIntrinsicValues() []string {
traceql.ScopedIntrinsicTraceDuration.String(),
traceql.IntrinsicEventName.String(),
traceql.IntrinsicEventTimeSinceStart.String(),
traceql.IntrinsicScopeName.String(),
traceql.IntrinsicScopeVersion.String(),
/* these are technically intrinsics that can be requested, but they are not generally of interest to a user
typing a query. for simplicity and clarity we are leaving them out of autocomplete
IntrinsicNestedSetLeft
Expand Down
2 changes: 2 additions & 0 deletions pkg/traceql/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ func ParseIdentifier(s string) (Attribute, error) {
return NewScopedAttribute(AttributeScopeResource, false, strings.TrimPrefix(s, "resource.")), nil
case strings.HasPrefix(s, "span."):
return NewScopedAttribute(AttributeScopeSpan, false, strings.TrimPrefix(s, "span.")), nil
case strings.HasPrefix(s, "scope."):
return NewScopedAttribute(AttributeScopeInstrumentation, false, strings.TrimPrefix(s, "scope.")), nil
case strings.HasPrefix(s, "event."):
return NewScopedAttribute(AttributeScopeEvent, false, strings.TrimPrefix(s, "event.")), nil
case strings.HasPrefix(s, "link."):
Expand Down
118 changes: 94 additions & 24 deletions tempodb/encoding/vparquet4/block_autocomplete.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,13 @@ func autocompleteIter(ctx context.Context, tr tagRequest, pf *parquet.File, opts
}
}

if len(catConditions.scope) > 0 || tr.keysRequested(traceql.AttributeScopeInstrumentation) {
currentIter, err = createDistinctScopeIterator(makeIter, tr, currentIter, catConditions.scope)
if err != nil {
return nil, errors.Wrap(err, "creating instrumentation iterator")
}
}

if len(catConditions.resource) > 0 || tr.keysRequested(traceql.AttributeScopeResource) {
currentIter, err = createDistinctResourceIterator(makeIter, tr, currentIter, catConditions.resource, dc)
if err != nil {
Expand Down Expand Up @@ -656,15 +663,7 @@ func createDistinctAttributeIterator(
valueIters = append(valueIters, makeIter(boolPath, orIfNeeded(boolPreds), "bool"))
}

scope := scopeFromDefinitionLevel(definitionLevel)
if definitionLevel == DefinitionLevelResourceSpansILSSpanEventAttrs {
switch keyPath {
case columnPathEventAttrKey:
scope = traceql.AttributeScopeEvent
case columnPathLinkAttrKey:
scope = traceql.AttributeScopeLink
}
}
scope := scopeFromDefinitionLevel(definitionLevel, keyPath)
if len(valueIters) > 0 || len(iters) > 0 || tr.keysRequested(scope) {
if len(valueIters) > 0 {
tagIter, err := parquetquery.NewLeftJoinIterator(
Expand Down Expand Up @@ -694,16 +693,7 @@ func createDistinctAttributeIterator(
}

func keyNameIterator(makeIter makeIterFn, definitionLevel int, keyPath string, attrIters []parquetquery.Iterator) (parquetquery.Iterator, error) {
scope := scopeFromDefinitionLevel(definitionLevel)
if definitionLevel == DefinitionLevelResourceSpansILSSpanEventAttrs {
switch keyPath {
case columnPathEventAttrKey:
scope = traceql.AttributeScopeEvent
case columnPathLinkAttrKey:
scope = traceql.AttributeScopeLink
}
}

scope := scopeFromDefinitionLevel(definitionLevel, keyPath)
if len(attrIters) == 0 {
return parquetquery.NewJoinIterator(
oneLevelUp(definitionLevel),
Expand Down Expand Up @@ -733,6 +723,71 @@ func oneLevelUp(definitionLevel int) int {
return definitionLevel
}

func createDistinctScopeIterator(
makeIter makeIterFn,
tr tagRequest,
primaryIter parquetquery.Iterator,
conditions []traceql.Condition,
) (parquetquery.Iterator, error) {
var (
iters []parquetquery.Iterator
genericConditions []traceql.Condition
)

for _, cond := range conditions {
// Intrinsic?
switch cond.Attribute.Intrinsic {
case traceql.IntrinsicScopeName:
pred, err := createStringPredicate(cond.Op, cond.Operands)
if err != nil {
return nil, err
}
selectAs := ""
if tr.tag == cond.Attribute {
selectAs = columnPathScopeName
}
iters = append(iters, makeIter(columnPathScopeName, pred, selectAs))
continue
case traceql.IntrinsicScopeVersion:
pred, err := createStringPredicate(cond.Op, cond.Operands)
if err != nil {
return nil, err
}
selectAs := ""
if tr.tag == cond.Attribute {
selectAs = columnPathScopeVersion
}
iters = append(iters, makeIter(columnPathScopeVersion, pred, selectAs))
continue
}
// Else: generic attribute lookup
genericConditions = append(genericConditions, cond)
}

attrIter, err := createDistinctAttributeIterator(makeIter, tr, genericConditions, DefinitionLevelInstrumentationScopeAttrs,
columnPathScopeAttrKey, columnPathScopeAttrString, columnPathScopeAttrInt, columnPathScopeAttrDouble, columnPathScopeAttrBool)
if err != nil {
return nil, errors.Wrap(err, "creating scope attribute iterator")
}

// if no intrinsics and no events then we can just return the attribute iterator
if len(iters) == 0 && primaryIter == nil {
return attrIter, nil
}

if attrIter != nil {
iters = append(iters, attrIter)
}

if primaryIter != nil {
iters = append(iters, primaryIter)
}

scopeCol := newDistinctValueCollector(mapScopeAttr, "scope")

return parquetquery.NewJoinIterator(DefinitionLevelInstrumentationScope, iters, scopeCol), nil
}

func createDistinctResourceIterator(
makeIter makeIterFn,
tr tagRequest,
Expand Down Expand Up @@ -1078,6 +1133,14 @@ func mapSpanAttr(e entry) traceql.Static {
return traceql.NewStaticNil()
}

func mapScopeAttr(e entry) traceql.Static {
switch e.Key {
case columnPathScopeName, columnPathScopeVersion:
return traceql.NewStaticString(unsafeToString(e.Value.ByteArray()))
}
return traceql.Static{}
}

func mapResourceAttr(e entry) traceql.Static {
switch e.Value.Kind() {
case parquet.Boolean:
Expand Down Expand Up @@ -1106,14 +1169,21 @@ func mapTraceAttr(e entry) traceql.Static {
return traceql.NewStaticNil()
}

func scopeFromDefinitionLevel(lvl int) traceql.AttributeScope {
func scopeFromDefinitionLevel(lvl int, keyPath string) traceql.AttributeScope {
switch lvl {
case DefinitionLevelResourceSpansILSSpan,
DefinitionLevelResourceSpansILSSpanAttrs:
case DefinitionLevelResourceSpansILSSpanAttrs:
return traceql.AttributeScopeSpan
case DefinitionLevelResourceAttrs,
DefinitionLevelResourceSpans:
case DefinitionLevelResourceAttrs:
return traceql.AttributeScopeResource
case DefinitionLevelInstrumentationScopeAttrs:
return traceql.AttributeScopeInstrumentation
case DefinitionLevelResourceSpansILSSpanEventAttrs:
switch keyPath {
case columnPathEventAttrKey:
return traceql.AttributeScopeEvent
default: // columnPathLinkAttrKey
return traceql.AttributeScopeLink
}
default:
return traceql.AttributeScopeNone
}
Expand Down
35 changes: 35 additions & 0 deletions tempodb/encoding/vparquet4/block_autocomplete_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues []string
expectedEventValues []string
expectedLinkValues []string
expectedScopeValues []string
}{
{
name: "no query - fall back to old search",
Expand All @@ -39,6 +40,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "generic-02", "resource-same"},
expectedEventValues: []string{"event-generic-01-01", "event-generic-02-01"},
expectedLinkValues: []string{"link-generic-01-01", "link-generic-02-01"},
expectedScopeValues: []string{"scope-attr-str-1", "scope-attr-str-2"},
},
{
name: "matches nothing",
Expand All @@ -47,6 +49,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{},
expectedEventValues: []string{},
expectedLinkValues: []string{},
expectedScopeValues: []string{},
},
// span
{
Expand All @@ -56,6 +59,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "resource-same"},
expectedEventValues: []string{"event-generic-01-01"},
expectedLinkValues: []string{"link-generic-01-01"},
expectedScopeValues: []string{"scope-attr-str-1"},
},
{
name: "well known span",
Expand All @@ -64,6 +68,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "resource-same"},
expectedEventValues: []string{"event-generic-01-01"},
expectedLinkValues: []string{"link-generic-01-01"},
expectedScopeValues: []string{"scope-attr-str-1"},
},
{
name: "generic span",
Expand All @@ -72,6 +77,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "resource-same"},
expectedEventValues: []string{"event-generic-01-01"},
expectedLinkValues: []string{"link-generic-01-01"},
expectedScopeValues: []string{"scope-attr-str-1"},
},
{
name: "match two spans",
Expand All @@ -80,6 +86,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"},
expectedEventValues: []string{"event-generic-01-01", "event-generic-02-01"},
expectedLinkValues: []string{"link-generic-01-01", "link-generic-02-01"},
expectedScopeValues: []string{"scope-attr-str-1", "scope-attr-str-2"},
},
// resource
{
Expand All @@ -89,6 +96,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "resource-same"},
expectedEventValues: []string{"event-generic-01-01"},
expectedLinkValues: []string{"link-generic-01-01"},
expectedScopeValues: []string{"scope-attr-str-1"},
},
{
name: "generic resource",
Expand All @@ -97,6 +105,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "resource-same"},
expectedEventValues: []string{"event-generic-01-01"},
expectedLinkValues: []string{"link-generic-01-01"},
expectedScopeValues: []string{"scope-attr-str-1"},
},
{
name: "match two resources",
Expand All @@ -105,6 +114,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"},
expectedEventValues: []string{"event-generic-01-01", "event-generic-02-01"},
expectedLinkValues: []string{"link-generic-01-01", "link-generic-02-01"},
expectedScopeValues: []string{"scope-attr-str-1", "scope-attr-str-2"},
},
// trace level match
{
Expand All @@ -114,6 +124,7 @@ func TestFetchTagNames(t *testing.T) {
expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"},
expectedEventValues: []string{"event-generic-01-01", "event-generic-02-01"},
expectedLinkValues: []string{"link-generic-01-01", "link-generic-02-01"},
expectedScopeValues: []string{"scope-attr-str-1", "scope-attr-str-2"},
},
}

Expand All @@ -137,6 +148,14 @@ func TestFetchTagNames(t *testing.T) {
},
ScopeSpans: []ScopeSpans{
{
Scope: InstrumentationScope{
Name: "scope-1",
Version: "version-1",
DroppedAttributesCount: 1,
Attrs: []Attribute{
attr("scope-attr-str-1", "scope-attr-1"),
},
},
Spans: []Span{
{
SpanID: []byte("0101"),
Expand Down Expand Up @@ -197,6 +216,14 @@ func TestFetchTagNames(t *testing.T) {
},
ScopeSpans: []ScopeSpans{
{
Scope: InstrumentationScope{
Name: "scope-2",
Version: "version-2",
DroppedAttributesCount: 1,
Attrs: []Attribute{
attr("scope-attr-str-2", "scope-attr-2"),
},
},
Spans: []Span{
{
SpanID: []byte("0201"),
Expand Down Expand Up @@ -246,11 +273,13 @@ func TestFetchTagNames(t *testing.T) {
traceql.AttributeScopeNone,
traceql.AttributeScopeEvent,
traceql.AttributeScopeLink,
traceql.AttributeScopeInstrumentation,
} {
expectedSpanValues := tc.expectedSpanValues
expectedResourceValues := tc.expectedResourceValues
expectedEventValues := tc.expectedEventValues
expectedLinkValues := tc.expectedLinkValues
expectedScopeValues := tc.expectedScopeValues

// add dedicated and well known columns to expected values. the code currently does not
// attempt to perfectly filter these, but instead adds them to the return if any values are present
Expand Down Expand Up @@ -282,6 +311,12 @@ func TestFetchTagNames(t *testing.T) {
}
}

if scope == traceql.AttributeScopeInstrumentation || scope == traceql.AttributeScopeNone {
if len(expectedScopeValues) > 0 {
expectedValues["scope"] = append(expectedValues["scope"], expectedScopeValues...)
}
}

t.Run(fmt.Sprintf("query: %s %s-%s", tc.name, tc.query, scope), func(t *testing.T) {
distinctAttrNames := collector.NewScopedDistinctString(0)
req, err := traceql.ExtractFetchSpansRequest(tc.query)
Expand Down
Loading

0 comments on commit 581fed2

Please sign in to comment.