Merge branch 'main' into updates-for-query-editor-11-2

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## main / unreleased
 
+* [ENHANCEMENT] TraceQL: Attribute iterators collect matched array values [#3867](https://github.com/grafana/tempo/pull/3867) (@electron0zero, @stoewer)
 * [ENHANCEMENT] Add bytes and spans received to usage stats [#3983](https://github.com/grafana/tempo/pull/3983) (@joe-elliott)
 
 # v2.6.0-rc.0
@@ -17,6 +18,7 @@
 * [FEATURE] TraecQL support for event attributes [#3708](https://github.com/grafana/tempo/pull/3748) (@ie-pham)
 * [FEATURE] TraceQL support for event:timeSinceStart [#3908](https://github.com/grafana/tempo/pull/3908) (@ie-pham)
 * [FEATURE] Autocomplete support for events and links [#3846](https://github.com/grafana/tempo/pull/3846) (@ie-pham)
+* [FEATURE] TraceQL support for instrumentation scope [#3967](https://github.com/grafana/tempo/pull/3967) (@ie-pham)
 * [FEATURE] Flush and query RF1 blocks for TraceQL metric queries [#3628](https://github.com/grafana/tempo/pull/3628) [#3691](https://github.com/grafana/tempo/pull/3691) [#3723](https://github.com/grafana/tempo/pull/3723) (@mapno)
 * [FEATURE] Add new compare() metrics function [#3695](https://github.com/grafana/tempo/pull/3695) (@mdisibio)
 * [FEATURE] Add new api `/api/metrics/query` for instant metrics queries [#3859](https://github.com/grafana/tempo/pull/3859) (@mdisibio)

docs/sources/tempo/traceql/_index.md

@@ -78,22 +78,24 @@ Custom attributes are prefixed with `.`, `span.` or `resource.`, whereas intrins
 
 The following table shows the current available scoped intrinsic fields:
 
-| **Field**               | **Type**    | **Definition**                                                  | **Example**                            |
-| ----------------------- | ----------- | --------------------------------------------------------------- | -------------------------------------- |
-| `span:status`           | status enum | status: error, ok, or unset                                     | `{ span:status = ok }`                 |
-| `span:statusMessage`    | string      | optional text accompanying the span status                      | `{ span:statusMessage = "Forbidden" }` |
-| `span:duration`         | duration    | end - start time of the span                                    | `{ span:duration > 100ms }`            |
-| `span:name`             | string      | operation or span name                                          | `{ span:name = "HTTP POST" }`          |
-| `span:kind`             | kind enum   | kind: server, client, producer, consumer, internal, unspecified | `{ span:kind = server }`               |
-| `span:id`               | string      | span id using hex string                                        | `{ span:id = "0000000000000001" }`     |
-| `trace:duration`        | duration    | max(end) - min(start) time of the spans in the trace            | `{ trace:duration > 100ms }`           |
-| `trace:rootName`        | string      | if it exists the name of the root span in the trace             | `{ trace:rootName = "HTTP GET" }`      |
-| `trace:rootService`     | string      | if it exists the service name of the root span in the trace     | `{ trace:rootServiceName = "gateway" }`|
-| `trace:id`              | string      | trace id using hex string                                       | `{ trace:id = "1234567890abcde" }`     |
-| `event:name`            | string      | name of event                                                   | `{ event:name = "exception" }`         |
-| `event:timeSinceStart`  | duration    | time of event in relation to the span start time                | `{ event:timeSinceStart > 2ms}`        |
-| `link:spanID`           | string      | link span id using hex string                                   | `{ link:spanID = "0000000000000001" }` |
-| `link:traceID`          | string      | link trace id using hex string                                  | `{ link:traceID = "1234567890abcde" }` |
+| **Field**                | **Type**    | **Definition**                                                  | **Example**                            |
+| ------------------------ | ----------- | --------------------------------------------------------------- | -------------------------------------- |
+| `span:status`            | status enum | status: error, ok, or unset                                     | `{ span:status = ok }`                 |
+| `span:statusMessage`     | string      | optional text accompanying the span status                      | `{ span:statusMessage = "Forbidden" }` |
+| `span:duration`          | duration    | end - start time of the span                                    | `{ span:duration > 100ms }`            |
+| `span:name`              | string      | operation or span name                                          | `{ span:name = "HTTP POST" }`          |
+| `span:kind`              | kind enum   | kind: server, client, producer, consumer, internal, unspecified | `{ span:kind = server }`               |
+| `span:id`                | string      | span id using hex string                                        | `{ span:id = "0000000000000001" }`     |
+| `trace:duration`         | duration    | max(end) - min(start) time of the spans in the trace            | `{ trace:duration > 100ms }`           |
+| `trace:rootName`         | string      | if it exists the name of the root span in the trace             | `{ trace:rootName = "HTTP GET" }`      |
+| `trace:rootService`      | string      | if it exists the service name of the root span in the trace     | `{ trace:rootServiceName = "gateway" }`|
+| `trace:id`               | string      | trace id using hex string                                       | `{ trace:id = "1234567890abcde" }`     |
+| `event:name`             | string      | name of event                                                   | `{ event:name = "exception" }`         |
+| `event:timeSinceStart`   | duration    | time of event in relation to the span start time                | `{ event:timeSinceStart > 2ms}`        |
+| `link:spanID`            | string      | link span id using hex string                                   | `{ link:spanID = "0000000000000001" }` |
+| `link:traceID`           | string      | link trace id using hex string                                  | `{ link:traceID = "1234567890abcde" }` |
+| `instrumentation:name`   | string      | instrumentation scope name                                      | `{ instrumentation:name = "grpc" }`    |
+| `instrumentation:version`| string      | instrumentation scope version                                   | `{ instrumentation:version = "1.0.0" }`|   
 
 `trace:duration`, `trace:rootName`, and `trace:rootService` are trace-level intrinsics and are the same for all spans in the same trace.
 Additionally, these intrinsics are significantly more performant because they have to inspect much less data then a span-level intrinsic.
@@ -111,7 +113,7 @@ This example searches all Kubernetes clusters called `service-name` that have a
 
 ### Attribute fields
 
-TraceQL has four different attribute scopes: span attributes, resource attributes, event attributes, and link attributes. By expanding a span in the Grafana UI, you can see both its span attributes (1 in the screenshot) and resource attributes (2 in the screenshot).
+TraceQL has five different attribute scopes: span attributes, resource attributes, event attributes, link attributes, instrumentation scope attributes. By expanding a span in the Grafana UI, you can see both its span attributes (1 in the screenshot) and resource attributes (2 in the screenshot).
 
 <p align="center"><img src="assets/span-resource-attributes.png" alt="Example of span and resource  attributes." /></p>
 
@@ -148,6 +150,11 @@ You can search for an attribute in your link:
 { link.opentracing.ref_type = "child_of" }
 ```
 
+Find instrumentation scope programming language:
+```
+{ instrumentation.language = "java" }
+```
+
 ### Unscoped attribute fields
 
 Attributes can be unscoped if you are unsure if the requested attribute exists on the span or resource.

integration/e2e/api_test.go

@@ -596,7 +596,7 @@ func callSearchTagsV2AndAssert(t *testing.T, svc *e2e.HTTPService, scope, query
 	if scope == "none" || scope == "" || scope == "intrinsic" {
 		expected.Scopes = append(expected.Scopes, &tempopb.SearchTagsV2Scope{
 			Name: "intrinsic",
-			Tags: []string{"duration", "event:name", "event:timeSinceStart", "kind", "name", "rootName", "rootServiceName", "span:duration", "span:kind", "span:name", "span:status", "span:statusMessage", "status", "statusMessage", "trace:duration", "trace:rootName", "trace:rootService", "traceDuration"},
+			Tags: []string{"duration", "event:name", "event:timeSinceStart", "instrumentation:name", "instrumentation:version", "kind", "name", "rootName", "rootServiceName", "span:duration", "span:kind", "span:name", "span:status", "span:statusMessage", "status", "statusMessage", "trace:duration", "trace:rootName", "trace:rootService", "traceDuration"},
 		})
 	}
 	sort.Slice(expected.Scopes, func(i, j int) bool { return expected.Scopes[i].Name < expected.Scopes[j].Name })

integration/e2e/multi_tenant_test.go

@@ -134,7 +134,7 @@ func testSearch(t *testing.T, tenant string, tenantSize int) {
 
 	tagsV2Resp, err := apiClient.SearchTagsV2()
 	require.NoError(t, err)
-	require.Equal(t, 4, len(tagsV2Resp.GetScopes())) // resource, span, event, intrinsics
+	require.Equal(t, 4, len(tagsV2Resp.GetScopes())) // resource, span, event, link, instrumentation intrinsics
 	for _, s := range tagsV2Resp.Scopes {
 		require.NotEmpty(t, s.Tags)
 	}

modules/ingester/instance_search_test.go

@@ -407,6 +407,24 @@ func testSearchTagsAndValuesV2(
 	sort.Strings(expectedLinkTagValues)
 	assert.Contains(t, tagsResp.TagNames, tagName)
 	assert.Equal(t, expectedLinkTagValues, tagValues)
+
+	// instrumentation scope attr
+
+	tagValuesResp, err = i.SearchTagValuesV2(ctx, &tempopb.SearchTagValuesRequest{
+		TagName: fmt.Sprintf("instrumentation.%s", tagName),
+		Query:   query,
+	})
+	require.NoError(t, err)
+
+	tagValues = make([]string, 0, len(tagValuesResp.TagValues))
+	for _, v := range tagValuesResp.TagValues {
+		tagValues = append(tagValues, v.Value)
+	}
+
+	sort.Strings(tagValues)
+	sort.Strings(expectedTagValues)
+	assert.Contains(t, tagsResp.TagNames, tagName)
+	assert.Equal(t, expectedTagValues, tagValues)
 }
 
 // TestInstanceSearchTagsSpecialCases tess that SearchTags errors on an unknown scope and
@@ -424,8 +442,11 @@ func TestInstanceSearchTagsSpecialCases(t *testing.T) {
 	require.Equal(
 		t,
 		[]string{
-			"duration", "event:name", "event:timeSinceStart", "kind", "name", "rootName", "rootServiceName",
-			"span:duration", "span:kind", "span:name", "span:status", "span:statusMessage", "status", "statusMessage",
+			"duration", "event:name", "event:timeSinceStart",
+			"instrumentation:name", "instrumentation:version",
+			"kind", "name", "rootName", "rootServiceName",
+			"span:duration", "span:kind", "span:name",
+			"span:status", "span:statusMessage", "status", "statusMessage",
 			"trace:duration", "trace:rootName", "trace:rootService", "traceDuration",
 		},
 		resp.TagNames,
@@ -562,6 +583,11 @@ func writeTracesForSearch(t *testing.T, i *instance, spanName, tagKey, tagValue
 		// add the time
 		for _, batch := range testTrace.ResourceSpans {
 			for _, ils := range batch.ScopeSpans {
+				ils.Scope = &v1.InstrumentationScope{
+					Name:       "scope-name",
+					Version:    "scope-version",
+					Attributes: []*v1.KeyValue{kv},
+				}
 				for _, span := range ils.Spans {
 					span.Name = spanName
 					span.StartTimeUnixNano = uint64(now.UnixNano())

pkg/search/util.go

@@ -74,6 +74,8 @@ func GetVirtualIntrinsicValues() []string {
 		traceql.ScopedIntrinsicTraceDuration.String(),
 		traceql.IntrinsicEventName.String(),
 		traceql.IntrinsicEventTimeSinceStart.String(),
+		traceql.IntrinsicInstrumentationName.String(),
+		traceql.IntrinsicInstrumentationVersion.String(),
 		/* these are technically intrinsics that can be requested, but they are not generally of interest to a user
 		   typing a query. for simplicity and clarity we are leaving them out of autocomplete
 			IntrinsicNestedSetLeft

pkg/traceql/ast.go

@@ -731,6 +731,52 @@ func (s Static) compare(o *Static) int {
 	}
 }
 
+type VisitFunc func(Static) bool // Return false to stop iteration
+
+// GetElements turns arrays into slice of Static elements to iterate over.
+func (s Static) GetElements(fn VisitFunc) error {
+	switch s.Type {
+	case TypeIntArray:
+		ints, _ := s.IntArray()
+		for _, n := range ints {
+			if !fn(NewStaticInt(n)) {
+				break // stop early if the callback returns false
+			}
+		}
+		return nil
+
+	case TypeFloatArray:
+		floats, _ := s.FloatArray()
+		for _, f := range floats {
+			if !fn(NewStaticFloat(f)) {
+				break
+			}
+		}
+		return nil
+
+	case TypeStringArray:
+		strs, _ := s.StringArray()
+		for _, str := range strs {
+			if !fn(NewStaticString(str)) {
+				break
+			}
+		}
+		return nil
+
+	case TypeBooleanArray:
+		bools, _ := s.BooleanArray()
+		for _, b := range bools {
+			if !fn(NewStaticBool(b)) {
+				break
+			}
+		}
+		return nil
+
+	default:
+		return fmt.Errorf("unsupported type")
+	}
+}
+
 func (s Static) Int() (int, bool) {
 	if s.Type != TypeInt {
 		return 0, false

pkg/traceql/ast_execute.go

@@ -381,7 +381,7 @@ func (o *BinaryOperation) execute(span Span) (Static, error) {
 	}
 
 	// if both sides are integers then do integer math, otherwise we can drop to the
-	// catch all below
+	// catch-all below
 	if lhsT == TypeInt && rhsT == TypeInt {
 		lhsN, _ := lhs.Int()
 		rhsN, _ := rhs.Int()
@@ -422,6 +422,39 @@ func (o *BinaryOperation) execute(span Span) (Static, error) {
 		}
 	}
 
+	if lhsT.isMatchingArrayElement(rhsT) {
+		// we only support boolean op in the arrays
+		if !o.Op.isBoolean() {
+			return NewStaticNil(), errors.ErrUnsupported
+		}
+
+		elemOp := &BinaryOperation{Op: o.Op, LHS: lhs, RHS: rhs}
+		arraySide := lhs
+		// to support symmetric operations
+		if rhsT.isArray() {
+			// for regex operations, TraceQL makes an assumption that RHS is the regex, and compiles it.
+			// we can support symmetric array operations by flipping the sides and executing the binary operation.
+			elemOp = &BinaryOperation{Op: getFlippedOp(o.Op), LHS: rhs, RHS: lhs}
+			arraySide = rhs
+		}
+
+		var res Static
+		err := arraySide.GetElements(func(elem Static) bool {
+			elemOp.LHS = elem
+			res, err = elemOp.execute(span)
+			if err != nil {
+				return false // stop iteration early if there's an error
+			}
+			match, ok := res.Bool()
+			return !(ok && match) // stop if a match is found
+		})
+		if err != nil {
+			return NewStaticNil(), err
+		}
+
+		return res, err
+	}
+
 	switch o.Op {
 	case OpAdd:
 		return NewStaticFloat(lhs.Float() + rhs.Float()), nil
@@ -452,6 +485,22 @@ func (o *BinaryOperation) execute(span Span) (Static, error) {
 	}
 }
 
+// getFlippedOp will return the flipped op, used when flipping the LHS and RHS of a BinaryOperation
+func getFlippedOp(op Operator) Operator {
+	switch op {
+	case OpGreater:
+		return OpLess
+	case OpGreaterEqual:
+		return OpLessEqual
+	case OpLess:
+		return OpGreater
+	case OpLessEqual:
+		return OpGreaterEqual
+	default:
+		return op
+	}
+}
+
 // why does this and the above exist?
 func binOp(op Operator, lhs, rhs Static) (bool, error) {
 	lhsT := lhs.impliedType()