TraceQL: Experimental "not structural" operators

CHANGELOG.md

@@ -6,6 +6,7 @@
 * [FEATURE] New encoding vParquet3 with support for dedicated attribute columns (@mapno, @stoewer) [#2649](https://github.com/grafana/tempo/pull/2649)
 * [FEATURE] Add filtering support to Generic Forwarding [#2742](https://github.com/grafana/tempo/pull/2742) (@Blinkuu)
 * [FEATURE] Add cli command to print out summary of large traces [#2775](https://github.com/grafana/tempo/pull/2775) (@ie-pham)
+* [FEATURE] Added not structural operators to TraceQL: !>, !<, and !~ [#2993](https://github.com/grafana/tempo/pull/2993) (@joe-elliott)
 * [CHANGE] Update Go to 1.21 [#2486](https://github.com/grafana/tempo/pull/2829) (@zalegrala)
 * [CHANGE] Make metrics-generator ingestion slack per tenant [#2589](https://github.com/grafana/tempo/pull/2589) (@ie-pham)
 * [CHANGE] Moved the tempo_ingester_traces_created_total metric to be incremented when a trace is cut to the wal [#2884](https://github.com/grafana/tempo/pull/2884) (@joe-elliott)

docs/sources/tempo/traceql/_index.md

@@ -208,20 +208,44 @@ The second expression returns no traces because it's impossible for a single spa
 
 ### Structural
 
-These spanset operators look at the structure of a trace and the relationship between the spans.
+These spanset operators look at the structure of a trace and the relationship between the spans. Structural operators ALWAYS return
+matches from the right hand side of the operator.
 
 - `{condA} >> {condB}` - The descendant operator (`>>`) looks for spans matching `{condB}` that are descendants of a span matching `{condA}`
 - `{condA} << {condB}` - The ancestor operator (`<<`) looks for spans matching `{condB}` that are ancestor of a span matching `{condA}`
 - `{condA} > {condB}` - The child operator (`>`) looks for spans matching `{condB}` that are direct child spans of a parent matching `{condA}`
 - `{condA} < {condB}` - The parent operator (`<`) looks for spans matching `{condB}` that are direct parent spans of a child matching `{condA}`
-- `{condA} ~ {condB}` - The sibling operator (`~`) checks that spans matching `{condA}` and `{condB}` are siblings of the same parent span.
+- `{condA} ~ {condB}` - The sibling operator (`~`) looks at spans matching `{condB}` that have at least one sibling matching `{condA}`.
 
 For example, to find a trace where a specific HTTP API interacted with a specific database:
 
 ```
 { span.http.url = "/path/of/api" } >> { span.db.name = "db-shard-001" }
 ```
 
+### Experimental Structural
+
+These spanset operators look at the structure of a trace and the relationship between the spans. They are marked experimental because they
+are known to sometimes return false positives. However, they can be very useful (see examples below). We encourage users to try them and give feedback.
+
+- `{condA} !>> {condB}` - The not-descendant operator (`!>>`) looks for spans matching `{condB}` that are not descendant spans of a parent matching `{condA}`
+- `{condA} !<< {condB}` - The not-ancestor operator (`!<<`) looks for spans matching `{condB}` that are not ancestor spans of a child matching `{condA}`
+- `{condA} !> {condB}` - The not-child operator (`!>`) looks for spans matching `{condB}` that are not direct child spans of a parent matching `{condA}`
+- `{condA} !< {condB}` - The not-parent operator (`!<`) looks for spans matching `{condB}` that are not direct parent spans of a child matching `{condA}`
+- `{condA} !~ {condB}` - The not-sibling operator (`!~`) looks that spans matching `{condB}` that do not have at least one sibling matching `{condA}`.
+
+For example, to find a trace with a leaf span in the service "foo":
+
+```
+{ } !< { resource.service.name = "foo" }
+```
+
+To find a span that is the last error in a series of cascading errors:
+
+```
+{ status = error } !< { status = error }
+```
+
 ## Aggregators
 
 So far, all of the example queries expressions have been about individual spans. You can use aggregate functions to ask questions about a set of spans. These currently consist of:

pkg/traceql/ast.go

@@ -234,15 +234,15 @@ type SpansetOperation struct {
 
 func (o SpansetOperation) extractConditions(request *FetchSpansRequest) {
 	switch o.Op {
-	case OpSpansetDescendant, OpSpansetAncestor:
+	case OpSpansetDescendant, OpSpansetAncestor, OpSpansetNotDescendant, OpSpansetNotAncestor:
 		request.Conditions = append(request.Conditions, Condition{
 			Attribute: NewIntrinsic(IntrinsicStructuralDescendant),
 		})
-	case OpSpansetChild, OpSpansetParent:
+	case OpSpansetChild, OpSpansetParent, OpSpansetNotChild, OpSpansetNotParent:
 		request.Conditions = append(request.Conditions, Condition{
 			Attribute: NewIntrinsic(IntrinsicStructuralChild),
 		})
-	case OpSpansetSibling:
+	case OpSpansetSibling, OpSpansetNotSibling:
 		request.Conditions = append(request.Conditions, Condition{
 			Attribute: NewIntrinsic(IntrinsicStructuralSibling),
 		})

pkg/traceql/ast_execute.go

@@ -85,6 +85,9 @@ func (o SpansetOperation) evaluate(input []*Spanset) (output []*Spanset, err err
 			return nil, err
 		}
 
+		var relFn func(l, r Span) bool
+		var falseForAll bool
+
 		switch o.Op {
 		case OpSpansetAnd:
 			if len(lhs) > 0 && len(rhs) > 0 {
@@ -100,76 +103,54 @@ func (o SpansetOperation) evaluate(input []*Spanset) (output []*Spanset, err err
 				output = append(output, matchingSpanset)
 			}
 
+		// relationship operators all set relFn which is used by below code
+		// to perform the operation
+		case OpSpansetNotDescendant:
+			falseForAll = true
+			fallthrough
 		case OpSpansetDescendant:
-			spans, err := o.joinSpansets(lhs, rhs, func(l, r Span) bool {
+			relFn = func(l, r Span) bool {
 				return r.DescendantOf(l)
-			})
-			if err != nil {
-				return nil, err
-			}
-
-			if len(spans) > 0 {
-				// Clone here to capture previously computed aggregates, grouped attrs, etc.
-				// Copy spans to new slice because of internal buffering.
-				matchingSpanset := input[i].clone()
-				matchingSpanset.Spans = append([]Span(nil), spans...)
-				output = append(output, matchingSpanset)
 			}
 
+		case OpSpansetNotAncestor:
+			falseForAll = true
+			fallthrough
 		case OpSpansetAncestor:
-			spans, err := o.joinSpansets(lhs, rhs, func(l, r Span) bool {
-				// In case of ancestor the lhs becomes descendant of rhs
+			relFn = func(l, r Span) bool {
 				return l.DescendantOf(r)
-			})
-			if err != nil {
-				return nil, err
-			}
-
-			if len(spans) > 0 {
-				// Clone here to capture previously computed aggregates, grouped attrs, etc.
-				// Copy spans to new slice because of internal buffering.
-				matchingSpanset := input[i].clone()
-				matchingSpanset.Spans = append([]Span(nil), spans...)
-				output = append(output, matchingSpanset)
 			}
 
+		case OpSpansetNotChild:
+			falseForAll = true
+			fallthrough
 		case OpSpansetChild:
-			spans, err := o.joinSpansets(lhs, rhs, func(l, r Span) bool {
+			relFn = func(l, r Span) bool {
 				return r.ChildOf(l)
-			})
-			if err != nil {
-				return nil, err
-			}
-
-			if len(spans) > 0 {
-				// Clone here to capture previously computed aggregates, grouped attrs, etc.
-				// Copy spans to new slice because of internal buffering.
-				matchingSpanset := input[i].clone()
-				matchingSpanset.Spans = append([]Span(nil), spans...)
-				output = append(output, matchingSpanset)
 			}
 
+		case OpSpansetNotParent:
+			falseForAll = true
+			fallthrough
 		case OpSpansetParent:
-			spans, err := o.joinSpansets(lhs, rhs, func(l, r Span) bool {
-				// In case of parent the lhs becomes child of rhs
+			relFn = func(l, r Span) bool {
 				return l.ChildOf(r)
-			})
-			if err != nil {
-				return nil, err
-			}
-
-			if len(spans) > 0 {
-				// Clone here to capture previously computed aggregates, grouped attrs, etc.
-				// Copy spans to new slice because of internal buffering.
-				matchingSpanset := input[i].clone()
-				matchingSpanset.Spans = append([]Span(nil), spans...)
-				output = append(output, matchingSpanset)
 			}
 
+		case OpSpansetNotSibling:
+			falseForAll = true
+			fallthrough
 		case OpSpansetSibling:
-			spans, err := o.joinSpansets(lhs, rhs, func(l, r Span) bool {
+			relFn = func(l, r Span) bool {
 				return r.SiblingOf(l)
-			})
+			}
+		default:
+			return nil, fmt.Errorf("spanset operation (%v) not supported", o.Op)
+		}
+
+		// if relFn was set up above we are doing a relationship operation.
+		if relFn != nil {
+			spans, err := o.joinSpansets(lhs, rhs, falseForAll, relFn)
 			if err != nil {
 				return nil, err
 			}
@@ -181,9 +162,6 @@ func (o SpansetOperation) evaluate(input []*Spanset) (output []*Spanset, err err
 				matchingSpanset.Spans = append([]Span(nil), spans...)
 				output = append(output, matchingSpanset)
 			}
-
-		default:
-			return nil, fmt.Errorf("spanset operation (%v) not supported", o.Op)
 		}
 	}
 
@@ -193,7 +171,7 @@ func (o SpansetOperation) evaluate(input []*Spanset) (output []*Spanset, err err
 // joinSpansets compares all pairwise combinations of the inputs and returns the right-hand side
 // where the eval callback returns true.  For now the behavior is only defined when there is exactly one
 // spanset on both sides and will return an error if multiple spansets are present.
-func (o *SpansetOperation) joinSpansets(lhs, rhs []*Spanset, eval func(l, r Span) bool) ([]Span, error) {
+func (o *SpansetOperation) joinSpansets(lhs, rhs []*Spanset, falseForAll bool, eval func(l, r Span) bool) ([]Span, error) {
 	if len(lhs) < 1 || len(rhs) < 1 {
 		return nil, nil
 	}
@@ -202,27 +180,34 @@ func (o *SpansetOperation) joinSpansets(lhs, rhs []*Spanset, eval func(l, r Span
 		return nil, errSpansetOperationMultiple
 	}
 
-	return o.joinSpansAndReturnRHS(lhs[0].Spans, rhs[0].Spans, eval), nil
+	return o.joinSpansAndReturnRHS(lhs[0].Spans, rhs[0].Spans, falseForAll, eval), nil
 }
 
 // joinSpansAndReturnRHS compares all pairwise combinations of the inputs and returns the right-hand side
 // spans where the eval callback returns true.  Uses and internal buffer and output is only valid until
 // the next call.  Destructively edits the RHS slice for performance.
-func (o *SpansetOperation) joinSpansAndReturnRHS(lhs, rhs []Span, eval func(l, r Span) bool) []Span {
+// falseForAll indicates that the spans on the RHS should only be returned if relFn returns
+// false for all on the LHS. otherwise spans on the RHS are returned if there are any matches on the lhs
+func (o *SpansetOperation) joinSpansAndReturnRHS(lhs, rhs []Span, falseForAll bool, eval func(l, r Span) bool) []Span {
 	if len(lhs) == 0 || len(rhs) == 0 {
 		return nil
 	}
 
 	o.matchingSpansBuffer = o.matchingSpansBuffer[:0]
 
 	for _, r := range rhs {
+		matches := false
 		for _, l := range lhs {
 			if eval(l, r) {
 				// Returns RHS
-				o.matchingSpansBuffer = append(o.matchingSpansBuffer, r)
+				matches = true
 				break
 			}
 		}
+		if matches && !falseForAll || // return RHS if there are any matches on the LHS
+			!matches && falseForAll { // return RHS if there are no matches on the LHS
+			o.matchingSpansBuffer = append(o.matchingSpansBuffer, r)
+		}
 	}
 
 	return o.matchingSpansBuffer

pkg/traceql/ast_execute_test.go

@@ -415,6 +415,81 @@ func TestSpansetOperationEvaluate(t *testing.T) {
 				}},
 			},
 		},
+		{
+			"{ } !< { .child }",
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{1}).WithAttrBool("parent", true).WithNestedSetInfo(0, 1, 4),
+					newMockSpan([]byte{2}).WithAttrBool("child", true).WithNestedSetInfo(1, 2, 3),
+					newMockSpan([]byte{2}).WithAttrBool("child", true).WithNestedSetInfo(0, 1, 4),
+				}},
+			},
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{2}).WithAttrBool("child", true).WithNestedSetInfo(1, 2, 3),
+				}},
+			},
+		},
+		{
+			"{ } !> { .parent }",
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{1}).WithAttrBool("parent", true).WithNestedSetInfo(0, 1, 4),
+					newMockSpan([]byte{1}).WithAttrBool("parent", true).WithNestedSetInfo(1, 2, 3),
+					newMockSpan([]byte{2}).WithAttrBool("child", true).WithNestedSetInfo(1, 2, 3),
+				}},
+			},
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{1}).WithAttrBool("parent", true).WithNestedSetInfo(0, 1, 4),
+				}},
+			},
+		},
+		{
+			"{ .child1 } !~ { .child2 }",
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{1}).WithAttrBool("child1", true).WithNestedSetInfo(1, 2, 3),
+					newMockSpan([]byte{1}).WithAttrBool("child2", true).WithNestedSetInfo(1, 4, 5),
+					newMockSpan([]byte{1}).WithAttrBool("child2", true).WithNestedSetInfo(4, 5, 6),
+				}},
+			},
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{1}).WithAttrBool("child2", true).WithNestedSetInfo(4, 5, 6),
+				}},
+			},
+		},
+		{
+			"{ } !<< { .child }",
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{1}).WithAttrBool("parent", true).WithNestedSetInfo(0, 1, 4),
+					newMockSpan([]byte{2}).WithAttrBool("child", true).WithNestedSetInfo(1, 2, 3),
+					newMockSpan([]byte{2}).WithAttrBool("child", true).WithNestedSetInfo(0, 1, 4),
+				}},
+			},
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{2}).WithAttrBool("child", true).WithNestedSetInfo(1, 2, 3),
+				}},
+			},
+		},
+		{
+			"{ } !>> { .parent }",
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{1}).WithAttrBool("parent", true).WithNestedSetInfo(0, 1, 4),
+					newMockSpan([]byte{1}).WithAttrBool("parent", true).WithNestedSetInfo(1, 2, 3),
+					newMockSpan([]byte{2}).WithAttrBool("child", true).WithNestedSetInfo(1, 2, 3),
+				}},
+			},
+			[]*Spanset{
+				{Spans: []Span{
+					newMockSpan([]byte{1}).WithAttrBool("parent", true).WithNestedSetInfo(0, 1, 4),
+				}},
+			},
+		},
 		{ // tests that child operators do not modify the spanset
 			"{ } > { } > { } > { }",
 			[]*Spanset{

pkg/traceql/ast_validate.go

@@ -179,11 +179,19 @@ func (o BinaryOperation) validate() error {
 		return fmt.Errorf("illegal operation for the given types: %s", o.String())
 	}
 
+	// this condition may not be possible to hit since it's not parseable.
+	// however, if we did somehow end up this situation, it would be good to return
+	// a reasonable error
 	switch o.Op {
 	case OpSpansetChild,
 		OpSpansetParent,
 		OpSpansetDescendant,
 		OpSpansetAncestor,
+		OpSpansetNotChild,
+		OpSpansetNotParent,
+		OpSpansetNotSibling,
+		OpSpansetNotAncestor,
+		OpSpansetNotDescendant,
 		OpSpansetSibling:
 		return newUnsupportedError(fmt.Sprintf("binary operation (%v)", o.Op))
 	}

pkg/traceql/enum_operators.go

@@ -30,6 +30,11 @@ const (
 	OpSpansetAnd
 	OpSpansetUnion
 	OpSpansetSibling
+	OpSpansetNotChild
+	OpSpansetNotParent
+	OpSpansetNotSibling
+	OpSpansetNotAncestor
+	OpSpansetNotDescendant
 )
 
 func (op Operator) isBoolean() bool {
@@ -163,6 +168,16 @@ func (op Operator) String() string {
 		return "~"
 	case OpSpansetUnion:
 		return "||"
+	case OpSpansetNotChild:
+		return "!>"
+	case OpSpansetNotParent:
+		return "!<"
+	case OpSpansetNotSibling:
+		return "!~"
+	case OpSpansetNotAncestor:
+		return "!<<"
+	case OpSpansetNotDescendant:
+		return "!>>"
 	}
 
 	return fmt.Sprintf("operator(%d)", op)