From be0148df23a6a4d5105e6070a2ed9a4a88b43b78 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 9 Sep 2024 18:06:30 +0530 Subject: [PATCH 01/17] feat: logsV4 initial refactoring --- pkg/query-service/app/logs/v3/json_filter.go | 24 +-- .../app/logs/v3/json_filter_test.go | 2 +- .../app/logs/v3/query_builder.go | 56 ++--- pkg/query-service/app/logs/v4/json_filter.go | 105 +++++++++ .../app/logs/v4/json_filter_test.go | 200 ++++++++++++++++++ .../app/logs/v4/query_builder.go | 66 ++++++ .../app/logs/v4/query_builder_test.go | 159 ++++++++++++++ pkg/query-service/constants/constants.go | 12 ++ pkg/query-service/model/v3/v3.go | 6 + pkg/query-service/utils/format.go | 22 ++ 10 files changed, 611 insertions(+), 41 deletions(-) create mode 100644 pkg/query-service/app/logs/v4/json_filter.go create mode 100644 pkg/query-service/app/logs/v4/json_filter_test.go create mode 100644 pkg/query-service/app/logs/v4/query_builder_test.go diff --git a/pkg/query-service/app/logs/v3/json_filter.go b/pkg/query-service/app/logs/v3/json_filter.go index 887baaab4c..d883b61797 100644 --- a/pkg/query-service/app/logs/v3/json_filter.go +++ b/pkg/query-service/app/logs/v3/json_filter.go @@ -20,7 +20,7 @@ const ( NGRAM_SIZE = 4 ) -var dataTypeMapping = map[string]string{ +var DataTypeMapping = map[string]string{ "string": STRING, "int64": INT64, "float64": FLOAT64, @@ -31,7 +31,7 @@ var dataTypeMapping = map[string]string{ "array(bool)": ARRAY_BOOL, } -var arrayValueTypeMapping = map[string]string{ +var ArrayValueTypeMapping = map[string]string{ "array(string)": "string", "array(int64)": "int64", "array(float64)": "float64", @@ -59,7 +59,7 @@ var jsonLogOperators = map[v3.FilterOperator]string{ v3.FilterOperatorNotHas: "NOT has(%s, %s)", } -func getPath(keyArr []string) string { +func GetPath(keyArr []string) string { path := []string{} for i := 0; i < len(keyArr); i++ { if strings.HasSuffix(keyArr[i], "[*]") { @@ -71,7 +71,7 @@ func getPath(keyArr []string) string { return strings.Join(path, ".") } -func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) { +func GetJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) { keyArr := strings.Split(key.Key, ".") // i.e it should be at least body.name, and not something like body if len(keyArr) < 2 { @@ -89,11 +89,11 @@ func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) ( var dataType string var ok bool - if dataType, ok = dataTypeMapping[string(key.DataType)]; !ok { + if dataType, ok = DataTypeMapping[string(key.DataType)]; !ok { return "", fmt.Errorf("unsupported dataType for JSON: %s", key.DataType) } - path := getPath(keyArr[1:]) + path := GetPath(keyArr[1:]) if isArray { return fmt.Sprintf("JSONExtract(JSON_QUERY(%s, '$.%s'), '%s')", keyArr[0], path, dataType), nil @@ -109,7 +109,7 @@ func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) ( } // takes the path and the values and generates where clauses for better usage of index -func getPathIndexFilter(path string) string { +func GetPathIndexFilter(path string) string { filters := []string{} keyArr := strings.Split(path, ".") if len(keyArr) < 2 { @@ -136,7 +136,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { dataType := item.Key.DataType isArray := false // check if its an array and handle it - if val, ok := arrayValueTypeMapping[string(item.Key.DataType)]; ok { + if val, ok := ArrayValueTypeMapping[string(item.Key.DataType)]; ok { if item.Operator != v3.FilterOperatorHas && item.Operator != v3.FilterOperatorNotHas { return "", fmt.Errorf("only has operator is supported for array") } @@ -144,7 +144,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { dataType = v3.AttributeKeyDataType(val) } - key, err := getJSONFilterKey(item.Key, item.Operator, isArray) + key, err := GetJSONFilterKey(item.Key, item.Operator, isArray) if err != nil { return "", err } @@ -164,7 +164,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { if logsOp, ok := jsonLogOperators[op]; ok { switch op { case v3.FilterOperatorExists, v3.FilterOperatorNotExists: - filter = fmt.Sprintf(logsOp, key, getPath(strings.Split(item.Key.Key, ".")[1:])) + filter = fmt.Sprintf(logsOp, key, GetPath(strings.Split(item.Key.Key, ".")[1:])) case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas: fmtVal := utils.ClickHouseFormattedValue(value) filter = fmt.Sprintf(logsOp, key, fmtVal) @@ -181,7 +181,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { filters := []string{} - pathFilter := getPathIndexFilter(item.Key.Key) + pathFilter := GetPathIndexFilter(item.Key.Key) if pathFilter != "" { filters = append(filters, pathFilter) } @@ -196,7 +196,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { // add exists check for non array items as default values of int/float/bool will corrupt the results if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) { - existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", getPath(strings.Split(item.Key.Key, ".")[1:])) + existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", GetPath(strings.Split(item.Key.Key, ".")[1:])) filter = fmt.Sprintf("%s AND %s", existsFilter, filter) } diff --git a/pkg/query-service/app/logs/v3/json_filter_test.go b/pkg/query-service/app/logs/v3/json_filter_test.go index 0a71cd67b2..060ba63707 100644 --- a/pkg/query-service/app/logs/v3/json_filter_test.go +++ b/pkg/query-service/app/logs/v3/json_filter_test.go @@ -140,7 +140,7 @@ var testGetJSONFilterKeyData = []struct { func TestGetJSONFilterKey(t *testing.T) { for _, tt := range testGetJSONFilterKeyData { Convey("testgetKey", t, func() { - columnName, err := getJSONFilterKey(tt.Key, tt.Operator, tt.IsArray) + columnName, err := GetJSONFilterKey(tt.Key, tt.Operator, tt.IsArray) if tt.Error { So(err, ShouldNotBeNil) } else { diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index 2aa56002ff..bd64b4d0e6 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -9,7 +9,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/utils" ) -var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ +var AggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ v3.AggregateOperatorP05: 0.05, v3.AggregateOperatorP10: 0.10, v3.AggregateOperatorP20: 0.20, @@ -21,7 +21,7 @@ var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ v3.AggregateOperatorP99: 0.99, } -var aggregateOperatorToSQLFunc = map[v3.AggregateOperator]string{ +var AggregateOperatorToSQLFunc = map[v3.AggregateOperator]string{ v3.AggregateOperatorAvg: "avg", v3.AggregateOperatorMax: "max", v3.AggregateOperatorMin: "min", @@ -53,7 +53,7 @@ var logOperators = map[v3.FilterOperator]string{ const BODY = "body" -func getClickhouseLogsColumnType(columnType v3.AttributeKeyType) string { +func GetClickhouseLogsColumnType(columnType v3.AttributeKeyType) string { if columnType == v3.AttributeKeyTypeTag { return "attributes" } @@ -83,7 +83,7 @@ func getClickhouseColumnName(key v3.AttributeKey) string { //if the key is present in the topLevelColumn then it will be only searched in those columns, //regardless if it is indexed/present again in resource or column attribute if !key.IsColumn { - columnType := getClickhouseLogsColumnType(key.Type) + columnType := GetClickhouseLogsColumnType(key.Type) columnDataType := getClickhouseLogsColumnDataType(key.DataType) clickhouseColumn = fmt.Sprintf("%s_%s_value[indexOf(%s_%s_key, '%s')]", columnType, columnDataType, columnType, columnDataType, key.Key) return clickhouseColumn @@ -114,7 +114,7 @@ func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attri return selectLabels } -func getSelectKeys(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { +func GetSelectKeys(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { var selectLabels []string if aggregatorOperator == v3.AggregateOperatorNoOp { return "" @@ -154,7 +154,7 @@ func GetExistsNexistsFilter(op v3.FilterOperator, item v3.FilterItem) string { } return fmt.Sprintf("%s_exists`=%v", strings.TrimSuffix(getClickhouseColumnName(item.Key), "`"), val) } - columnType := getClickhouseLogsColumnType(item.Key.Type) + columnType := GetClickhouseLogsColumnType(item.Key.Type) columnDataType := getClickhouseLogsColumnDataType(item.Key.DataType) return fmt.Sprintf(logOperators[op], columnType, columnDataType, item.Key.Key) } @@ -224,7 +224,7 @@ func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey, // add group by conditions to filter out log lines which doesn't have the key for _, attr := range groupBy { if !attr.IsColumn { - columnType := getClickhouseLogsColumnType(attr.Type) + columnType := GetClickhouseLogsColumnType(attr.Type) columnDataType := getClickhouseLogsColumnDataType(attr.DataType) conditions = append(conditions, fmt.Sprintf("has(%s_%s_key, '%s')", columnType, columnDataType, attr.Key)) } else if attr.Type != v3.AttributeKeyTypeUnspecified { @@ -258,7 +258,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build selectLabels := getSelectLabels(mq.AggregateOperator, mq.GroupBy) - having := having(mq.Having) + having := Having(mq.Having) if having != "" { having = " having " + having } @@ -288,10 +288,10 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build // we dont need value for first query // going with this route as for a cleaner approach on implementation if graphLimitQtype == constants.FirstQueryGraphLimit { - queryTmpl = "SELECT " + getSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + queryTmpl + ")" + queryTmpl = "SELECT " + GetSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + queryTmpl + ")" } - groupBy := groupByAttributeKeyTags(panelType, graphLimitQtype, mq.GroupBy...) + groupBy := GroupByAttributeKeyTags(panelType, graphLimitQtype, mq.GroupBy...) if panelType != v3.PanelTypeList && groupBy != "" { groupBy = " group by " + groupBy } @@ -301,7 +301,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build } if graphLimitQtype == constants.SecondQueryGraphLimit { - filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", getSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "#LIMIT_PLACEHOLDER)" + filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", GetSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "#LIMIT_PLACEHOLDER)" } aggregationKey := "" @@ -329,7 +329,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build rate = rate / 60.0 } - op := fmt.Sprintf("%s(%s)/%f", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, rate) + op := fmt.Sprintf("%s(%s)/%f", AggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, rate) query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil case @@ -342,11 +342,11 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build v3.AggregateOperatorP90, v3.AggregateOperatorP95, v3.AggregateOperatorP99: - op := fmt.Sprintf("quantile(%v)(%s)", aggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey) + op := fmt.Sprintf("quantile(%v)(%s)", AggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey) query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax: - op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey) + op := fmt.Sprintf("%s(%s)", AggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey) query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorCount: @@ -394,7 +394,7 @@ func groupBy(panelType v3.PanelType, graphLimitQtype string, tags ...string) str return strings.Join(tags, ",") } -func groupByAttributeKeyTags(panelType v3.PanelType, graphLimitQtype string, tags ...v3.AttributeKey) string { +func GroupByAttributeKeyTags(panelType v3.PanelType, graphLimitQtype string, tags ...v3.AttributeKey) string { groupTags := []string{} for _, tag := range tags { groupTags = append(groupTags, "`"+tag.Key+"`") @@ -446,7 +446,7 @@ func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags [] return str } -func having(items []v3.Having) string { +func Having(items []v3.Having) string { // aggregate something and filter on that aggregate var having []string for _, item := range items { @@ -455,7 +455,7 @@ func having(items []v3.Having) string { return strings.Join(having, " AND ") } -func reduceQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator v3.AggregateOperator) (string, error) { +func ReduceQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator v3.AggregateOperator) (string, error) { // the timestamp picked is not relevant here since the final value used is show the single // chart with just the query value. switch reduceTo { @@ -475,14 +475,14 @@ func reduceQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator v return query, nil } -func addLimitToQuery(query string, limit uint64) string { +func AddLimitToQuery(query string, limit uint64) string { if limit == 0 { return query } return fmt.Sprintf("%s LIMIT %d", query, limit) } -func addOffsetToQuery(query string, offset uint64) string { +func AddOffsetToQuery(query string, offset uint64) string { return fmt.Sprintf("%s OFFSET %d", query, offset) } @@ -492,7 +492,7 @@ type Options struct { PreferRPM bool } -func isOrderByTs(orderBy []v3.OrderBy) bool { +func IsOrderByTs(orderBy []v3.OrderBy) bool { if len(orderBy) == 1 && (orderBy[0].Key == constants.TIMESTAMP || orderBy[0].ColumnName == constants.TIMESTAMP) { return true } @@ -523,7 +523,7 @@ func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.Pan if err != nil { return "", err } - query = addLimitToQuery(query, mq.Limit) + query = AddLimitToQuery(query, mq.Limit) return query, nil } else if options.GraphLimitQtype == constants.SecondQueryGraphLimit { @@ -539,7 +539,7 @@ func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.Pan return "", err } if panelType == v3.PanelTypeValue { - query, err = reduceQuery(query, mq.ReduceTo, mq.AggregateOperator) + query, err = ReduceQuery(query, mq.ReduceTo, mq.AggregateOperator) } if panelType == v3.PanelTypeList { @@ -550,21 +550,21 @@ func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.Pan if mq.PageSize > 0 { if mq.Limit > 0 && mq.Offset+mq.PageSize > mq.Limit { - query = addLimitToQuery(query, mq.Limit-mq.Offset) + query = AddLimitToQuery(query, mq.Limit-mq.Offset) } else { - query = addLimitToQuery(query, mq.PageSize) + query = AddLimitToQuery(query, mq.PageSize) } // add offset to the query only if it is not orderd by timestamp. - if !isOrderByTs(mq.OrderBy) { - query = addOffsetToQuery(query, mq.Offset) + if !IsOrderByTs(mq.OrderBy) { + query = AddOffsetToQuery(query, mq.Offset) } } else { - query = addLimitToQuery(query, mq.Limit) + query = AddLimitToQuery(query, mq.Limit) } } else if panelType == v3.PanelTypeTable { - query = addLimitToQuery(query, mq.Limit) + query = AddLimitToQuery(query, mq.Limit) } return query, err diff --git a/pkg/query-service/app/logs/v4/json_filter.go b/pkg/query-service/app/logs/v4/json_filter.go new file mode 100644 index 0000000000..cde88e748a --- /dev/null +++ b/pkg/query-service/app/logs/v4/json_filter.go @@ -0,0 +1,105 @@ +package v4 + +import ( + "fmt" + "strings" + + logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) + +var jsonLogOperators = map[v3.FilterOperator]string{ + v3.FilterOperatorEqual: "=", + v3.FilterOperatorNotEqual: "!=", + v3.FilterOperatorLessThan: "<", + v3.FilterOperatorLessThanOrEq: "<=", + v3.FilterOperatorGreaterThan: ">", + v3.FilterOperatorGreaterThanOrEq: ">=", + v3.FilterOperatorLike: "LIKE", + v3.FilterOperatorNotLike: "NOT LIKE", + v3.FilterOperatorContains: "LIKE", + v3.FilterOperatorNotContains: "NOT LIKE", + v3.FilterOperatorRegex: "match(%s, %s)", + v3.FilterOperatorNotRegex: "NOT match(%s, %s)", + v3.FilterOperatorIn: "IN", + v3.FilterOperatorNotIn: "NOT IN", + v3.FilterOperatorExists: "JSON_EXISTS(%s, '$.%s')", + v3.FilterOperatorNotExists: "NOT JSON_EXISTS(%s, '$.%s')", + v3.FilterOperatorHas: "has(%s, %s)", + v3.FilterOperatorNotHas: "NOT has(%s, %s)", +} + +func GetJSONFilter(item v3.FilterItem) (string, error) { + + dataType := item.Key.DataType + isArray := false + // check if its an array and handle it + if val, ok := logsV3.ArrayValueTypeMapping[string(item.Key.DataType)]; ok { + if item.Operator != v3.FilterOperatorHas && item.Operator != v3.FilterOperatorNotHas { + return "", fmt.Errorf("only has operator is supported for array") + } + isArray = true + dataType = v3.AttributeKeyDataType(val) + } + + key, err := logsV3.GetJSONFilterKey(item.Key, item.Operator, isArray) + if err != nil { + return "", err + } + + // non array + op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator)))) + + var value interface{} + if op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists { + value, err = utils.ValidateAndCastValue(item.Value, dataType) + if err != nil { + return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err) + } + } + + var filter string + if logsOp, ok := jsonLogOperators[op]; ok { + switch op { + case v3.FilterOperatorExists, v3.FilterOperatorNotExists: + filter = fmt.Sprintf(logsOp, key, logsV3.GetPath(strings.Split(item.Key.Key, ".")[1:])) + case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas: + fmtVal := utils.ClickHouseFormattedValue(value) + filter = fmt.Sprintf(logsOp, key, fmtVal) + case v3.FilterOperatorContains, v3.FilterOperatorNotContains: + val := utils.QuoteEscapedString(fmt.Sprintf("%v", item.Value)) + filter = fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, val) + default: + fmtVal := utils.ClickHouseFormattedValue(value) + filter = fmt.Sprintf("%s %s %s", key, logsOp, fmtVal) + } + } else { + return "", fmt.Errorf("unsupported operator: %s", op) + } + + filters := []string{} + + pathFilter := logsV3.GetPathIndexFilter(item.Key.Key) + if pathFilter != "" { + filters = append(filters, pathFilter) + } + if op == v3.FilterOperatorContains || + op == v3.FilterOperatorEqual || + op == v3.FilterOperatorHas { + val, ok := item.Value.(string) + if ok && len(val) >= logsV3.NGRAM_SIZE { + filters = append(filters, fmt.Sprintf("lower(body) like lower('%%%s%%')", utils.QuoteEscapedString(strings.ToLower(val)))) + } + } + + // add exists check for non array items as default values of int/float/bool will corrupt the results + if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) { + existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", logsV3.GetPath(strings.Split(item.Key.Key, ".")[1:])) + filter = fmt.Sprintf("%s AND %s", existsFilter, filter) + } + + filters = append(filters, filter) + + return strings.Join(filters, " AND "), nil +} diff --git a/pkg/query-service/app/logs/v4/json_filter_test.go b/pkg/query-service/app/logs/v4/json_filter_test.go new file mode 100644 index 0000000000..c8b2e44847 --- /dev/null +++ b/pkg/query-service/app/logs/v4/json_filter_test.go @@ -0,0 +1,200 @@ +package v4 + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +var testGetJSONFilterData = []struct { + Name string + FilterItem v3.FilterItem + Filter string + Error bool +}{ + { + Name: "Array membership string", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.requestor_list[*]", + DataType: "array(string)", + IsJSON: true, + }, + Operator: "has", + Value: "index_service", + }, + Filter: "lower(body) like lower('%requestor_list%') AND lower(body) like lower('%index_service%') AND has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service')", + }, + { + Name: "Array membership int64", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.int_numbers[*]", + DataType: "array(int64)", + IsJSON: true, + }, + Operator: "has", + Value: 2, + }, + Filter: "lower(body) like lower('%int_numbers%') AND has(JSONExtract(JSON_QUERY(body, '$.\"int_numbers\"[*]'), '" + logsV3.ARRAY_INT64 + "'), 2)", + }, + { + Name: "Array membership float64", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.nested_num[*].float_nums[*]", + DataType: "array(float64)", + IsJSON: true, + }, + Operator: "nhas", + Value: 2.2, + }, + Filter: "lower(body) like lower('%nested_num%float_nums%') AND NOT has(JSONExtract(JSON_QUERY(body, '$.\"nested_num\"[*].\"float_nums\"[*]'), '" + logsV3.ARRAY_FLOAT64 + "'), 2.200000)", + }, + { + Name: "Array membership bool", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.bool[*]", + DataType: "array(bool)", + IsJSON: true, + }, + Operator: "has", + Value: true, + }, + Filter: "lower(body) like lower('%bool%') AND has(JSONExtract(JSON_QUERY(body, '$.\"bool\"[*]'), '" + logsV3.ARRAY_BOOL + "'), true)", + }, + { + Name: "eq operator", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.message", + DataType: "string", + IsJSON: true, + }, + Operator: "=", + Value: "hello", + }, + Filter: "lower(body) like lower('%message%') AND lower(body) like lower('%hello%') AND JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') = 'hello'", + }, + { + Name: "eq operator number", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.status", + DataType: "int64", + IsJSON: true, + }, + Operator: "=", + Value: 1, + }, + Filter: "lower(body) like lower('%status%') AND JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + logsV3.INT64 + "') = 1", + }, + { + Name: "neq operator number", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.status", + DataType: "float64", + IsJSON: true, + }, + Operator: "=", + Value: 1.1, + }, + Filter: "lower(body) like lower('%status%') AND JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + logsV3.FLOAT64 + "') = 1.100000", + }, + { + Name: "eq operator bool", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.boolkey", + DataType: "bool", + IsJSON: true, + }, + Operator: "=", + Value: true, + }, + Filter: "lower(body) like lower('%boolkey%') AND JSON_EXISTS(body, '$.\"boolkey\"') AND JSONExtract(JSON_VALUE(body, '$.\"boolkey\"'), '" + logsV3.BOOL + "') = true", + }, + { + Name: "greater than operator", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.status", + DataType: "int64", + IsJSON: true, + }, + Operator: ">", + Value: 1, + }, + Filter: "lower(body) like lower('%status%') AND JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + logsV3.INT64 + "') > 1", + }, + { + Name: "regex operator", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.message", + DataType: "string", + IsJSON: true, + }, + Operator: "regex", + Value: "a*", + }, + Filter: "lower(body) like lower('%message%') AND JSON_EXISTS(body, '$.\"message\"') AND match(JSON_VALUE(body, '$.\"message\"'), 'a*')", + }, + { + Name: "contains operator", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.message", + DataType: "string", + IsJSON: true, + }, + Operator: "contains", + Value: "a", + }, + Filter: "lower(body) like lower('%message%') AND JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') LIKE '%a%'", + }, + { + Name: "contains operator with quotes", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.message", + DataType: "string", + IsJSON: true, + }, + Operator: "contains", + Value: "hello 'world'", + }, + Filter: "lower(body) like lower('%message%') AND lower(body) like lower('%hello \\'world\\'%') AND JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') LIKE '%hello \\'world\\'%'", + }, + { + Name: "exists", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.message", + DataType: "string", + IsJSON: true, + }, + Operator: "exists", + Value: "", + }, + Filter: "lower(body) like lower('%message%') AND JSON_EXISTS(body, '$.\"message\"')", + }, +} + +func TestGetJSONFilter(t *testing.T) { + for _, tt := range testGetJSONFilterData { + Convey("testGetJSONFilter", t, func() { + filter, err := GetJSONFilter(tt.FilterItem) + if tt.Error { + So(err, ShouldNotBeNil) + } else { + So(err, ShouldBeNil) + So(filter, ShouldEqual, tt.Filter) + } + }) + } +} diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index 08024756bd..e753de73eb 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -1,7 +1,13 @@ package v4 import ( + "fmt" + "strings" + + logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" ) var logOperators = map[v3.FilterOperator]string{ @@ -29,3 +35,63 @@ const ( DISTRIBUTED_LOGS_V2_RESOURCE = "distributed_logs_v2_resource" NANOSECOND = 1000000000 ) + +func getClickhouseLogsColumnDataType(columnDataType v3.AttributeKeyDataType) string { + if columnDataType == v3.AttributeKeyDataTypeFloat64 || columnDataType == v3.AttributeKeyDataTypeInt64 { + return "number" + } + if columnDataType == v3.AttributeKeyDataTypeBool { + return "bool" + } + return "string" +} + +func getClickhouseKey(key v3.AttributeKey) string { + // check if it is a top level static field + if _, ok := constants.StaticFieldsLogsV3[key.Key]; ok && key.Type == v3.AttributeKeyTypeUnspecified { + return key.Key + } + + //if the key is present in the topLevelColumn then it will be only searched in those columns, + //regardless if it is indexed/present again in resource or column attribute + if !key.IsColumn { + columnType := logsV3.GetClickhouseLogsColumnType(key.Type) + columnDataType := getClickhouseLogsColumnDataType(key.DataType) + return fmt.Sprintf("%s_%s['%s']", columnType, columnDataType, key.Key) + } + + // materialized column created from query + return utils.GetClickhouseColumnNameV2(string(key.Type), string(key.DataType), key.Key) +} + +func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { + var selectLabels string + if aggregatorOperator == v3.AggregateOperatorNoOp { + selectLabels = "" + } else { + for _, tag := range groupBy { + columnName := getClickhouseKey(tag) + selectLabels += fmt.Sprintf(" %s as `%s`,", columnName, tag.Key) + } + } + return selectLabels +} + +func getExistsNexistsFilter(op v3.FilterOperator, item v3.FilterItem) string { + if _, ok := constants.StaticFieldsLogsV3[item.Key.Key]; ok && item.Key.Type == v3.AttributeKeyTypeUnspecified { + // no exists filter for static fields as they exists everywhere + // TODO(nitya): Think what we can do here + return "" + } else if item.Key.IsColumn { + // get filter for materialized columns + val := true + if op == v3.FilterOperatorNotExists { + val = false + } + return fmt.Sprintf("%s_exists`=%v", strings.TrimSuffix(getClickhouseKey(item.Key), "`"), val) + } + // filter for non materialized attributes + columnType := logsV3.GetClickhouseLogsColumnType(item.Key.Type) + columnDataType := getClickhouseLogsColumnDataType(item.Key.DataType) + return fmt.Sprintf(logOperators[op], columnType, columnDataType, item.Key.Key) +} diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go new file mode 100644 index 0000000000..95eebf1b00 --- /dev/null +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -0,0 +1,159 @@ +package v4 + +import ( + "testing" + + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func Test_getClickhouseKey(t *testing.T) { + type args struct { + key v3.AttributeKey + } + tests := []struct { + name string + args args + want string + }{ + { + name: "attribute", + args: args{ + key: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + }, + want: "attributes_string['user_name']", + }, + { + name: "resource", + args: args{ + key: v3.AttributeKey{Key: "servicename", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, + }, + want: "resources_string['servicename']", + }, + { + name: "selected field", + args: args{ + key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + want: "`attribute_number_bytes`", + }, + { + name: "selected field resource", + args: args{ + key: v3.AttributeKey{Key: "servicename", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: true}, + }, + want: "`resource_string_servicename`", + }, + { + name: "top level key", + args: args{ + key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString}, + }, + want: "trace_id", + }, + { + name: "name with -", + args: args{ + key: v3.AttributeKey{Key: "service-name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + want: "`attribute_string_service-name`", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getClickhouseKey(tt.args.key); got != tt.want { + t.Errorf("getClickhouseKey() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getSelectLabels(t *testing.T) { + type args struct { + aggregatorOperator v3.AggregateOperator + groupBy []v3.AttributeKey + } + tests := []struct { + name string + args args + want string + }{ + { + name: "count", + args: args{ + aggregatorOperator: v3.AggregateOperatorCount, + groupBy: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: " attributes_string['user_name'] as `user_name`,", + }, + { + name: "multiple group by", + args: args{ + aggregatorOperator: v3.AggregateOperatorCount, + groupBy: []v3.AttributeKey{ + {Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "service_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: true}, + }, + }, + want: " attributes_string['user_name'] as `user_name`, `resource_string_service_name` as `service_name`,", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getSelectLabels(tt.args.aggregatorOperator, tt.args.groupBy); got != tt.want { + t.Errorf("getSelectLabels() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getExistsNexistsFilter(t *testing.T) { + type args struct { + op v3.FilterOperator + item v3.FilterItem + } + tests := []struct { + name string + args args + want string + }{ + { + name: "exists", + args: args{ + op: v3.FilterOperatorExists, + item: v3.FilterItem{Key: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: "mapContains(attributes_string, 'user_name')", + }, + { + name: "not exists", + args: args{ + op: v3.FilterOperatorNotExists, + item: v3.FilterItem{Key: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: "not mapContains(attributes_string, 'user_name')", + }, + { + name: "exists mat column", + args: args{ + op: v3.FilterOperatorExists, + item: v3.FilterItem{Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + }, + want: "`attribute_number_bytes_exists`=true", + }, + { + name: "exists top level column", + args: args{ + op: v3.FilterOperatorExists, + item: v3.FilterItem{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified}}, + }, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getExistsNexistsFilter(tt.args.op, tt.args.item); got != tt.want { + t.Errorf("getExistsNexistsFilter() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 8c8a038f2f..26824813a8 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -311,6 +311,12 @@ const ( "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64," + "CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool," + "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string " + LogsSQLSelectV2 = "SELECT " + + "timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body," + + "attributes_string," + + "attributes_number," + + "attributes_bool," + + "resources_string " TracesExplorerViewSQLSelectWithSubQuery = "WITH subQuery AS (SELECT distinct on (traceID) traceID, durationNano, " + "serviceName, name FROM %s.%s WHERE parentSpanID = '' AND %s %s ORDER BY durationNano DESC " TracesExplorerViewSQLSelectQuery = "SELECT subQuery.serviceName, subQuery.name, count() AS " + @@ -375,6 +381,12 @@ var StaticFieldsLogsV3 = map[string]v3.AttributeKey{ Type: v3.AttributeKeyTypeUnspecified, IsColumn: true, }, + "__attrs": { + Key: "__attrs", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + }, } const SigNozOrderByValue = "#SIGNOZ_VALUE" diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 0f04375198..7a60dfdb21 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -1289,3 +1289,9 @@ type URLShareableOptions struct { Format string `json:"format"` SelectColumns []AttributeKey `json:"selectColumns"` } + +type LogQBOptions struct { + GraphLimitQtype string + IsLivetailQuery bool + PreferRPM bool +} diff --git a/pkg/query-service/utils/format.go b/pkg/query-service/utils/format.go index c623d3e8e0..9f06d712de 100644 --- a/pkg/query-service/utils/format.go +++ b/pkg/query-service/utils/format.go @@ -272,6 +272,28 @@ func GetClickhouseColumnName(typeName string, dataType, field string) string { return colName } +func GetClickhouseColumnNameV2(typeName string, dataType, field string) string { + if typeName == string(v3.AttributeKeyTypeTag) { + typeName = constants.Attributes + } + + if typeName != string(v3.AttributeKeyTypeResource) { + typeName = typeName[:len(typeName)-1] + } + + dataType = strings.ToLower(dataType) + + if dataType == "int64" || dataType == "float64" { + dataType = "number" + } + + // if name contains . replace it with `$$` + field = strings.ReplaceAll(field, ".", "$$") + + colName := fmt.Sprintf("`%s_%s_%s`", strings.ToLower(typeName), dataType, field) + return colName +} + // GetEpochNanoSecs takes epoch and returns it in ns func GetEpochNanoSecs(epoch int64) int64 { temp := epoch From 065111c44f15c002c1257525311f35b3752c1c43 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 9 Sep 2024 23:00:00 +0530 Subject: [PATCH 02/17] feat: filter_query builder with tests added --- .../app/logs/v4/query_builder.go | 126 +++++++- .../app/logs/v4/query_builder_test.go | 281 ++++++++++++++++++ pkg/query-service/utils/format.go | 2 +- 3 files changed, 407 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index e753de73eb..8fce2204fb 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -61,7 +61,7 @@ func getClickhouseKey(key v3.AttributeKey) string { } // materialized column created from query - return utils.GetClickhouseColumnNameV2(string(key.Type), string(key.DataType), key.Key) + return "`" + utils.GetClickhouseColumnNameV2(string(key.Type), string(key.DataType), key.Key) + "`" } func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { @@ -95,3 +95,127 @@ func getExistsNexistsFilter(op v3.FilterOperator, item v3.FilterItem) string { columnDataType := getClickhouseLogsColumnDataType(item.Key.DataType) return fmt.Sprintf(logOperators[op], columnType, columnDataType, item.Key.Key) } + +func buildAttributeFilter(item v3.FilterItem) (string, error) { + // check if the user is searching for value in all attributes + key := item.Key.Key + op := v3.FilterOperator(strings.ToLower(string(item.Operator))) + + var value interface{} + var err error + if op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists { + value, err = utils.ValidateAndCastValue(item.Value, item.Key.DataType) + if err != nil { + return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err) + } + } + + // TODO(nitya): as of now __attrs is only supports attributes_string. Discuss more on this + // also for eq and contains as now it does a exact match + if key == "__attrs" { + if (op != v3.FilterOperatorEqual && op != v3.FilterOperatorContains) || item.Key.DataType != v3.AttributeKeyDataTypeString { + return "", fmt.Errorf("only = operator and string data type is supported for __attrs") + } + val := utils.ClickHouseFormattedValue(item.Value) + return fmt.Sprintf("has(mapValues(attributes_string), %s)", val), nil + } + + keyName := getClickhouseKey(item.Key) + fmtVal := utils.ClickHouseFormattedValue(value) + + if logsOp, ok := logOperators[op]; ok { + switch op { + case v3.FilterOperatorExists, v3.FilterOperatorNotExists: + return getExistsNexistsFilter(op, item), nil + case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex: + + return fmt.Sprintf(logsOp, keyName, fmtVal), nil + case v3.FilterOperatorContains, v3.FilterOperatorNotContains: + val := utils.QuoteEscapedStringForContains(fmt.Sprintf("%s", item.Value)) + // for body the contains is case insensitive + if keyName == BODY { + return fmt.Sprintf("lower(%s) %s lower('%%%s%%')", keyName, logsOp, val), nil + } else { + return fmt.Sprintf("%s %s '%%%s%%'", keyName, logsOp, val), nil + } + default: + // for use lower for like and ilike + if op == v3.FilterOperatorLike || op == v3.FilterOperatorNotLike { + if keyName == BODY { + keyName = fmt.Sprintf("lower(%s)", keyName) + fmtVal = fmt.Sprintf("lower(%s)", fmtVal) + } + } + return fmt.Sprintf("%s %s %s", keyName, logsOp, fmtVal), nil + } + } else { + return "", fmt.Errorf("unsupported operator: %s", op) + } +} + +func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey) (string, error) { + var conditions []string + + if fs == nil || len(fs.Items) == 0 { + return "", nil + } + + for _, item := range fs.Items { + // skip if it's a resource attribute + if item.Key.Type == v3.AttributeKeyTypeResource { + continue + } + + // if the filter is json filter + if item.Key.IsJSON { + filter, err := GetJSONFilter(item) + if err != nil { + return "", err + } + conditions = append(conditions, filter) + continue + } + + // generate the filter + filter, err := buildAttributeFilter(item) + if err != nil { + return "", err + } + conditions = append(conditions, filter) + + // add extra condition for map contains + // by default clickhouse is not able to utilize indexes for keys with all operators. + // mapContains forces the use of index. + op := v3.FilterOperator(strings.ToLower(string(item.Operator))) + if item.Key.IsColumn == false && op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists { + conditions = append(conditions, getExistsNexistsFilter(v3.FilterOperatorExists, item)) + } + } + + // add group by conditions to filter out log lines which doesn't have the key + for _, attr := range groupBy { + // skip if it's a resource attribute + if attr.Type == v3.AttributeKeyTypeResource { + continue + } + + if !attr.IsColumn { + columnType := logsV3.GetClickhouseLogsColumnType(attr.Type) + columnDataType := getClickhouseLogsColumnDataType(attr.DataType) + conditions = append(conditions, fmt.Sprintf("mapContains(%s_%s, '%s')", columnType, columnDataType, attr.Key)) + } else if attr.Type != v3.AttributeKeyTypeUnspecified { + // for materialzied columns and not the top level static fields + name := utils.GetClickhouseColumnNameV2(string(attr.Type), string(attr.DataType), attr.Key) + conditions = append(conditions, fmt.Sprintf("`%s_exists`=true", name)) + } + } + + // add conditions for aggregate attribute + if aggregateAttribute.Key != "" && aggregateAttribute.Type != v3.AttributeKeyTypeResource { + existsFilter := getExistsNexistsFilter(v3.FilterOperatorExists, v3.FilterItem{Key: aggregateAttribute}) + conditions = append(conditions, existsFilter) + } + + queryString := strings.Join(conditions, " AND ") + return queryString, nil +} diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go index 95eebf1b00..c555881d1e 100644 --- a/pkg/query-service/app/logs/v4/query_builder_test.go +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -157,3 +157,284 @@ func Test_getExistsNexistsFilter(t *testing.T) { }) } } + +func Test_buildAttributeFilter(t *testing.T) { + type args struct { + item v3.FilterItem + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "build attribute filter", + args: args{ + item: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + }, + Operator: v3.FilterOperatorEqual, + Value: "test", + }, + }, + want: "resources_string['service.name'] = 'test'", + wantErr: false, + }, + { + name: "test for value search across all attributes", + args: args{ + item: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "__attrs", + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorContains, + Value: "test", + }, + }, + want: "has(mapValues(attributes_string), 'test')", + }, + { + name: "build attribute filter exists", + args: args{ + item: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + }, + Operator: v3.FilterOperatorExists, + }, + }, + want: "mapContains(resources_string, 'service.name')", + wantErr: false, + }, + { + name: "build attribute filter regex", + args: args{ + item: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + }, + Operator: v3.FilterOperatorRegex, + Value: "^test", + }, + }, + want: "match(resources_string['service.name'], '^test')", + }, + { + name: "build attribute filter contains", + args: args{ + item: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + }, + Operator: v3.FilterOperatorContains, + Value: "test", + }, + }, + want: "resources_string['service.name'] LIKE '%test%'", + }, + { + name: "build attribute filter contains- body", + args: args{ + item: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + Operator: v3.FilterOperatorContains, + Value: "test", + }, + }, + want: "lower(body) LIKE lower('%test%')", + }, + { + name: "build attribute filter like", + args: args{ + item: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + }, + Operator: v3.FilterOperatorLike, + Value: "test", + }, + }, + want: "resources_string['service.name'] LIKE 'test'", + }, + { + name: "build attribute filter like-body", + args: args{ + item: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + Operator: v3.FilterOperatorLike, + Value: "test", + }, + }, + want: "lower(body) LIKE lower('test')", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := buildAttributeFilter(tt.args.item) + if (err != nil) != tt.wantErr { + t.Errorf("buildAttributeFilter() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("buildAttributeFilter() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_buildLogsTimeSeriesFilterQuery(t *testing.T) { + type args struct { + fs *v3.FilterSet + groupBy []v3.AttributeKey + aggregateAttribute v3.AttributeKey + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "build logs time series filter query", + args: args{ + fs: &v3.FilterSet{ + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: v3.FilterOperatorEqual, + Value: "test", + }, + { + Key: v3.AttributeKey{ + Key: "method", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: v3.FilterOperatorEqual, + Value: "GET", + }, + }, + }, + }, + want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + "AND attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method')", + }, + { + name: "build logs time series filter query with group by and aggregate attribute", + args: args{ + fs: &v3.FilterSet{ + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: v3.FilterOperatorEqual, + Value: "test", + }, + }, + }, + groupBy: []v3.AttributeKey{ + { + Key: "user_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + aggregateAttribute: v3.AttributeKey{ + Key: "test", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + "AND mapContains(attributes_string, 'user_name') AND mapContains(attributes_string, 'test')", + }, + { + name: "build logs time series filter query with multiple group by and aggregate attribute", + args: args{ + fs: &v3.FilterSet{ + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: v3.FilterOperatorEqual, + Value: "test", + }, + }, + }, + groupBy: []v3.AttributeKey{ + { + Key: "user_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + { + Key: "host", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + }, + { + Key: "method", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + }, + { + Key: "trace_id", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + }, + aggregateAttribute: v3.AttributeKey{ + Key: "test", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + "AND mapContains(attributes_string, 'user_name') AND `attribute_string_method_exists`=true AND mapContains(attributes_string, 'test')", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := buildLogsTimeSeriesFilterQuery(tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute) + if (err != nil) != tt.wantErr { + t.Errorf("buildLogsTimeSeriesFilterQuery() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("buildLogsTimeSeriesFilterQuery() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/query-service/utils/format.go b/pkg/query-service/utils/format.go index 9f06d712de..e9b7a0b7e3 100644 --- a/pkg/query-service/utils/format.go +++ b/pkg/query-service/utils/format.go @@ -290,7 +290,7 @@ func GetClickhouseColumnNameV2(typeName string, dataType, field string) string { // if name contains . replace it with `$$` field = strings.ReplaceAll(field, ".", "$$") - colName := fmt.Sprintf("`%s_%s_%s`", strings.ToLower(typeName), dataType, field) + colName := fmt.Sprintf("%s_%s_%s", strings.ToLower(typeName), dataType, field) return colName } From 934155bd489ffe416da264b7ca7323760cb14347 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 10:45:23 +0530 Subject: [PATCH 03/17] feat: all functions of v4 refactored --- .../app/logs/v4/query_builder.go | 305 +++++++++ .../app/logs/v4/query_builder_test.go | 635 +++++++++++++++++- pkg/query-service/constants/constants.go | 8 +- 3 files changed, 941 insertions(+), 7 deletions(-) diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index 8fce2204fb..53909542b5 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -219,3 +219,308 @@ func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey, queryString := strings.Join(conditions, " AND ") return queryString, nil } + +// orderBy returns a string of comma separated tags for order by clause +// if there are remaining items which are not present in tags they are also added +// if the order is not specified, it defaults to ASC +func orderBy(panelType v3.PanelType, items []v3.OrderBy, tagLookup map[string]struct{}) []string { + var orderBy []string + + for _, item := range items { + if item.ColumnName == constants.SigNozOrderByValue { + orderBy = append(orderBy, fmt.Sprintf("value %s", item.Order)) + } else if _, ok := tagLookup[item.ColumnName]; ok { + orderBy = append(orderBy, fmt.Sprintf("`%s` %s", item.ColumnName, item.Order)) + } else if panelType == v3.PanelTypeList { + attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn} + name := getClickhouseKey(attr) + if item.IsColumn { + name = "`" + name + "`" + } + orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) + } + } + return orderBy +} + +func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags []v3.AttributeKey) string { + + tagLookup := map[string]struct{}{} + for _, v := range tags { + tagLookup[v.Key] = struct{}{} + } + + orderByArray := orderBy(panelType, items, tagLookup) + + if len(orderByArray) == 0 { + if panelType == v3.PanelTypeList { + orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC") + } else { + orderByArray = append(orderByArray, "value DESC") + } + } + + str := strings.Join(orderByArray, ",") + return str +} + +func generateAggregateClause(aggOp v3.AggregateOperator, + aggKey string, + step int64, + preferRPM bool, + timeFilter string, + whereClause string, + groupBy string, + having string, + orderBy string, +) (string, error) { + queryTmpl := " %s as value from signoz_logs." + DISTRIBUTED_LOGS_V2 + + " where " + timeFilter + "%s" + + "%s%s" + + "%s" + switch aggOp { + case v3.AggregateOperatorRate: + rate := float64(step) + if preferRPM { + rate = rate / 60.0 + } + + op := fmt.Sprintf("count(%s)/%f", aggKey, rate) + query := fmt.Sprintf(queryTmpl, op, whereClause, groupBy, having, orderBy) + return query, nil + case + v3.AggregateOperatorRateSum, + v3.AggregateOperatorRateMax, + v3.AggregateOperatorRateAvg, + v3.AggregateOperatorRateMin: + rate := float64(step) + if preferRPM { + rate = rate / 60.0 + } + + op := fmt.Sprintf("%s(%s)/%f", logsV3.AggregateOperatorToSQLFunc[aggOp], aggKey, rate) + query := fmt.Sprintf(queryTmpl, op, whereClause, groupBy, having, orderBy) + return query, nil + case + v3.AggregateOperatorP05, + v3.AggregateOperatorP10, + v3.AggregateOperatorP20, + v3.AggregateOperatorP25, + v3.AggregateOperatorP50, + v3.AggregateOperatorP75, + v3.AggregateOperatorP90, + v3.AggregateOperatorP95, + v3.AggregateOperatorP99: + op := fmt.Sprintf("quantile(%v)(%s)", logsV3.AggregateOperatorToPercentile[aggOp], aggKey) + query := fmt.Sprintf(queryTmpl, op, whereClause, groupBy, having, orderBy) + return query, nil + case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax: + op := fmt.Sprintf("%s(%s)", logsV3.AggregateOperatorToSQLFunc[aggOp], aggKey) + query := fmt.Sprintf(queryTmpl, op, whereClause, groupBy, having, orderBy) + return query, nil + case v3.AggregateOperatorCount: + op := "toFloat64(count(*))" + query := fmt.Sprintf(queryTmpl, op, whereClause, groupBy, having, orderBy) + return query, nil + case v3.AggregateOperatorCountDistinct: + op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggKey) + query := fmt.Sprintf(queryTmpl, op, whereClause, groupBy, having, orderBy) + return query, nil + default: + return "", fmt.Errorf("unsupported aggregate operator") + } +} + +func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.BuilderQuery, graphLimitQtype string, preferRPM bool) (string, error) { + // timerange will be sent in epoch millisecond + logsStart := utils.GetEpochNanoSecs(start) + logsEnd := utils.GetEpochNanoSecs(end) + + // -1800 this is added so that the bucket start considers all the fingerprints. + bucketStart := logsStart/NANOSECOND - 1800 + bucketEnd := logsEnd / NANOSECOND + + // timestamp filter , bucket_start filter is added for primary key + timeFilter := fmt.Sprintf("(timestamp >= %d AND timestamp <= %d) AND (ts_bucket_start >= %d AND ts_bucket_start <= %d)", logsStart, logsEnd, bucketStart, bucketEnd) + + // build the where clause for main table + filterSubQuery, err := buildLogsTimeSeriesFilterQuery(mq.Filters, mq.GroupBy, mq.AggregateAttribute) + if err != nil { + return "", err + } + if filterSubQuery != "" { + filterSubQuery = " AND " + filterSubQuery + } + + // build the where clause for resource table + resourceSubQuery, err := buildResourceSubQuery(bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute) + if err != nil { + return "", err + } + // join both the filter clauses + if resourceSubQuery != "" { + filterSubQuery = filterSubQuery + " AND (resource_fingerprint GLOBAL IN " + resourceSubQuery + ")" + } + + // get the select labels + selectLabels := getSelectLabels(mq.AggregateOperator, mq.GroupBy) + + // get the order by clause + orderBy := orderByAttributeKeyTags(panelType, mq.OrderBy, mq.GroupBy) + if panelType != v3.PanelTypeList && orderBy != "" { + orderBy = " order by " + orderBy + } + + // if noop create the query and return + if mq.AggregateOperator == v3.AggregateOperatorNoOp { + // with noop any filter or different order by other than ts will use new table + sqlSelect := constants.LogsSQLSelectV2 + queryTmpl := sqlSelect + "from signoz_logs.%s where %s%s order by %s" + query := fmt.Sprintf(queryTmpl, DISTRIBUTED_LOGS_V2, timeFilter, filterSubQuery, orderBy) + return query, nil + // ---- NOOP ends here ---- + } + + // ---- FOR aggregation queries ---- + + // get the having conditions + having := logsV3.Having(mq.Having) + if having != "" { + having = " having " + having + } + + // get the group by clause + groupBy := logsV3.GroupByAttributeKeyTags(panelType, graphLimitQtype, mq.GroupBy...) + if panelType != v3.PanelTypeList && groupBy != "" { + groupBy = " group by " + groupBy + } + + // get the aggregation key + aggregationKey := "" + if mq.AggregateAttribute.Key != "" { + aggregationKey = getClickhouseKey(mq.AggregateAttribute) + } + + // for limit queries, there are two queries formed + // in the second query we need to add the placeholder so that first query can be placed + if graphLimitQtype == constants.SecondQueryGraphLimit { + filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", logsV3.GetSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "#LIMIT_PLACEHOLDER)" + } + + aggClause, err := generateAggregateClause(mq.AggregateOperator, aggregationKey, step, preferRPM, timeFilter, filterSubQuery, groupBy, having, orderBy) + if err != nil { + return "", err + } + + var queryTmplPrefix string + if graphLimitQtype == constants.FirstQueryGraphLimit { + queryTmplPrefix = "SELECT" + } else if panelType == v3.PanelTypeTable { + queryTmplPrefix = + "SELECT now() as ts," + // step or aggregate interval is whole time period in case of table panel + step = (utils.GetEpochNanoSecs(end) - utils.GetEpochNanoSecs(start)) / NANOSECOND + } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { + // Select the aggregate value for interval + queryTmplPrefix = + fmt.Sprintf("SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL %d SECOND) AS ts,", step) + } + + query := queryTmplPrefix + selectLabels + aggClause + + // for limit query this is the first query, + // we don't the the aggregation value here as we are just concerned with the names of group by + // for applying the limit + if graphLimitQtype == constants.FirstQueryGraphLimit { + query = "SELECT " + logsV3.GetSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + query + ")" + } + return query, nil +} + +func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) { + filterSubQuery, err := buildLogsTimeSeriesFilterQuery(mq.Filters, mq.GroupBy, v3.AttributeKey{}) + if err != nil { + return "", err + } + + switch mq.AggregateOperator { + case v3.AggregateOperatorNoOp: + query := constants.LogsSQLSelectV2 + "from signoz_logs." + DISTRIBUTED_LOGS_V2 + " where " + if len(filterSubQuery) > 0 { + query = query + filterSubQuery + " AND " + } + + return query, nil + default: + return "", fmt.Errorf("unsupported aggregate operator in live tail") + } +} + +// PrepareLogsQuery prepares the query for logs +func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) { + + // adjust the start and end time to the step interval + // NOTE: Disabling this as it's creating confusion between charts and actual data + // if panelType != v3.PanelTypeList { + // start = start - (start % (mq.StepInterval * 1000)) + // end = end - (end % (mq.StepInterval * 1000)) + // } + + if options.IsLivetailQuery { + query, err := buildLogsLiveTailQuery(mq) + if err != nil { + return "", err + } + return query, nil + } else if options.GraphLimitQtype == constants.FirstQueryGraphLimit { + // give me just the group_by names (no values) + query, err := buildLogsQuery(panelType, start, end, mq.StepInterval, mq, options.GraphLimitQtype, options.PreferRPM) + if err != nil { + return "", err + } + query = logsV3.AddLimitToQuery(query, mq.Limit) + + return query, nil + } else if options.GraphLimitQtype == constants.SecondQueryGraphLimit { + query, err := buildLogsQuery(panelType, start, end, mq.StepInterval, mq, options.GraphLimitQtype, options.PreferRPM) + if err != nil { + return "", err + } + return query, nil + } + + query, err := buildLogsQuery(panelType, start, end, mq.StepInterval, mq, options.GraphLimitQtype, options.PreferRPM) + if err != nil { + return "", err + } + if panelType == v3.PanelTypeValue { + query, err = logsV3.ReduceQuery(query, mq.ReduceTo, mq.AggregateOperator) + } + + if panelType == v3.PanelTypeList { + // check if limit exceeded + if mq.Limit > 0 && mq.Offset >= mq.Limit { + return "", fmt.Errorf("max limit exceeded") + } + + if mq.PageSize > 0 { + if mq.Limit > 0 && mq.Offset+mq.PageSize > mq.Limit { + query = logsV3.AddLimitToQuery(query, mq.Limit-mq.Offset) + } else { + query = logsV3.AddLimitToQuery(query, mq.PageSize) + } + + // add offset to the query only if it is not orderd by timestamp. + if !logsV3.IsOrderByTs(mq.OrderBy) { + query = logsV3.AddOffsetToQuery(query, mq.Offset) + } + + } else { + query = logsV3.AddLimitToQuery(query, mq.Limit) + } + } else if panelType == v3.PanelTypeTable { + query = logsV3.AddLimitToQuery(query, mq.Limit) + } + + return query, err +} diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go index c555881d1e..69c8a7093e 100644 --- a/pkg/query-service/app/logs/v4/query_builder_test.go +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -3,6 +3,7 @@ package v4 import ( "testing" + "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" ) @@ -341,7 +342,7 @@ func Test_buildLogsTimeSeriesFilterQuery(t *testing.T) { }, }, }, - want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + want: " AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + "AND attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method')", }, { @@ -373,7 +374,7 @@ func Test_buildLogsTimeSeriesFilterQuery(t *testing.T) { Type: v3.AttributeKeyTypeTag, }, }, - want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + want: " AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + "AND mapContains(attributes_string, 'user_name') AND mapContains(attributes_string, 'test')", }, { @@ -421,7 +422,7 @@ func Test_buildLogsTimeSeriesFilterQuery(t *testing.T) { Type: v3.AttributeKeyTypeTag, }, }, - want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + want: " AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + "AND mapContains(attributes_string, 'user_name') AND `attribute_string_method_exists`=true AND mapContains(attributes_string, 'test')", }, } @@ -438,3 +439,631 @@ func Test_buildLogsTimeSeriesFilterQuery(t *testing.T) { }) } } + +func Test_orderByAttributeKeyTags(t *testing.T) { + type args struct { + panelType v3.PanelType + items []v3.OrderBy + tags []v3.AttributeKey + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Test 1", + args: args{ + panelType: v3.PanelTypeGraph, + items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: constants.SigNozOrderByValue, + Order: "desc", + }, + }, + tags: []v3.AttributeKey{ + {Key: "name"}, + }, + }, + want: "`name` asc,value desc", + }, + { + name: "Test Graph item not present in tag", + args: args{ + panelType: v3.PanelTypeGraph, + items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: "bytes", + Order: "asc", + }, + { + ColumnName: "method", + Order: "asc", + }, + }, + tags: []v3.AttributeKey{ + {Key: "name"}, + {Key: "bytes"}, + }, + }, + want: "`name` asc,`bytes` asc", + }, + { + name: "Test panel list", + args: args{ + panelType: v3.PanelTypeList, + items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: constants.SigNozOrderByValue, + Order: "asc", + }, + { + ColumnName: "bytes", + Order: "asc", + }, + }, + tags: []v3.AttributeKey{ + {Key: "name"}, + {Key: "bytes"}, + }, + }, + want: "`name` asc,value asc,`bytes` asc", + }, + { + name: "test 4", + args: args{ + panelType: v3.PanelTypeList, + items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: constants.SigNozOrderByValue, + Order: "asc", + }, + { + ColumnName: "response_time", + Order: "desc", + Key: "response_time", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + }, + tags: []v3.AttributeKey{ + {Key: "name"}, + {Key: "value"}, + }, + }, + want: "`name` asc,value asc,attributes_string['response_time'] desc", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := orderByAttributeKeyTags(tt.args.panelType, tt.args.items, tt.args.tags); got != tt.want { + t.Errorf("orderByAttributeKeyTags() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_generateAggregateClause(t *testing.T) { + type args struct { + op v3.AggregateOperator + aggKey string + step int64 + preferRPM bool + timeFilter string + whereClause string + groupBy string + having string + orderBy string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test rate", + args: args{ + op: v3.AggregateOperatorRate, + aggKey: "test", + step: 60, + preferRPM: false, + timeFilter: "(timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458)", + whereClause: " AND attributes_string['service.name'] = 'test'", + groupBy: " group by `user_name`", + having: "", + orderBy: " order by `user_name` desc", + }, + want: " count(test)/60.000000 as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND " + + "(ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['service.name'] = 'test' " + + "group by `user_name` order by `user_name` desc", + }, + { + name: "test P10 with all args", + args: args{ + op: v3.AggregateOperatorRate, + aggKey: "test", + step: 60, + preferRPM: false, + timeFilter: "(timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458)", + whereClause: " AND attributes_string['service.name'] = 'test'", + groupBy: " group by `user_name`", + having: " having value > 10", + orderBy: " order by `user_name` desc", + }, + want: " count(test)/60.000000 as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND " + + "(ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['service.name'] = 'test' group by `user_name` having value > 10 order by " + + "`user_name` desc", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := generateAggregateClause(tt.args.op, tt.args.aggKey, tt.args.step, tt.args.preferRPM, tt.args.timeFilter, tt.args.whereClause, tt.args.groupBy, tt.args.having, tt.args.orderBy) + if (err != nil) != tt.wantErr { + t.Errorf("generateAggreagteClause() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("generateAggreagteClause() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_buildLogsQuery(t *testing.T) { + type args struct { + panelType v3.PanelType + start int64 + end int64 + step int64 + mq *v3.BuilderQuery + graphLimitQtype string + preferRPM bool + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "build logs query", + args: args{ + panelType: v3.PanelTypeTable, + start: 1680066360726210000, + end: 1680066458000000000, + step: 1000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorCount, + Filters: &v3.FilterSet{ + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: v3.FilterOperatorEqual, + Value: "test", + }, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "user_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + OrderBy: []v3.OrderBy{ + { + ColumnName: "user_name", + Order: "desc", + }, + }, + }, + }, + want: "SELECT now() as ts, attributes_string['user_name'] as `user_name`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 " + + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') AND mapContains(attributes_string, 'user_name') " + + "group by `user_name` order by `user_name` desc", + }, + { + name: "build logs query noop", + args: args{ + panelType: v3.PanelTypeList, + start: 1680066360726210000, + end: 1680066458000000000, + step: 1000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorNoOp, + Filters: &v3.FilterSet{ + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: v3.FilterOperatorEqual, + Value: "test", + }, + }, + }, + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, + }, + }, + }, + want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string " + + "from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') order by timestamp desc", + }, + { + name: "build logs query with all args", + args: args{ + panelType: v3.PanelTypeGraph, + start: 1680066360726210000, + end: 1680066458000000000, + step: 60, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorAvg, + AggregateAttribute: v3.AttributeKey{ + Key: "duration", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeFloat64, + }, + Filters: &v3.FilterSet{ + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + }, + Operator: v3.FilterOperatorEqual, + Value: "test", + }, + { + Key: v3.AttributeKey{ + Key: "duration", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeTag, + }, + Operator: v3.FilterOperatorGreaterThan, + Value: 1000, + }, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "host", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + }, + }, + OrderBy: []v3.OrderBy{ + { + ColumnName: "host", + Order: "desc", + }, + }, + }, + }, + want: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, resources_string['host'] as `host`, avg(attributes_number['duration']) as value " + + "from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "AND attributes_number['duration'] > 1000.000000 AND mapContains(attributes_number, 'duration') AND mapContains(attributes_number, 'duration') AND " + + "(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) " + + "AND simpleJSONExtractString(labels, 'service.name') = 'test' AND labels like '%service.name%test%' AND ( (simpleJSONHas(labels, 'host') AND labels like '%host%') ))) " + + "group by `host`,ts order by `host` desc", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := buildLogsQuery(tt.args.panelType, tt.args.start, tt.args.end, tt.args.step, tt.args.mq, tt.args.graphLimitQtype, tt.args.preferRPM) + if (err != nil) != tt.wantErr { + t.Errorf("buildLogsQuery() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("buildLogsQuery() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPrepareLogsQuery(t *testing.T) { + type args struct { + start int64 + end int64 + queryType v3.QueryType + panelType v3.PanelType + mq *v3.BuilderQuery + options v3.LogQBOptions + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy", + args: args{ + start: 1680066360726210000, + end: 1680066458000000000, + panelType: v3.PanelTypeTable, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "body.requestor_list[*]", + DataType: "array(string)", + IsJSON: true, + }, + Operator: "has", + Value: "index_service", + }, + }, + }, + GroupBy: []v3.AttributeKey{ + {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + }, + OrderBy: []v3.OrderBy{ + {ColumnName: "name", Order: "DESC"}, + }, + }, + }, + want: "SELECT now() as ts, attributes_string['name'] as `name`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 " + + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "AND lower(body) like lower('%requestor_list%') AND lower(body) like lower('%index_service%') AND " + + "has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service') AND mapContains(attributes_string, 'name') " + + "group by `name` order by `name` DESC", + }, + { + name: "Test TS with limit- first", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeGraph, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + Limit: 10, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + }, + want: "SELECT `method` from (SELECT attributes_string['method'] as `method`, toFloat64(count(distinct(attributes_string['name']))) as value from signoz_logs.distributed_logs_v2 " + + "where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['method'] = 'GET' " + + "AND mapContains(attributes_string, 'method') AND mapContains(attributes_string, 'method') AND mapContains(attributes_string, 'name') group by `method` order by value DESC) LIMIT 10", + }, + { + name: "Test TS with limit- second", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeGraph, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + Limit: 2, + }, + options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, + }, + want: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string['method'] as `method`, toFloat64(count(distinct(attributes_string['name']))) as value " + + "from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "AND attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND mapContains(attributes_string, 'method') " + + "AND mapContains(attributes_string, 'name') AND (`method`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `method`,ts order by value DESC", + }, + { + name: "Live Tail Query", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeList, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + }, + options: v3.LogQBOptions{IsLivetailQuery: true}, + }, + want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string " + + "from signoz_logs.distributed_logs_v2 where attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND ", + }, + { + name: "Live Tail Query W/O filter", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeList, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + }, + options: v3.LogQBOptions{IsLivetailQuery: true}, + }, + want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string " + + "from signoz_logs.distributed_logs_v2 where ", + }, + { + name: "Table query with limit", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeTable, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + Limit: 10, + }, + }, + want: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) order by value DESC LIMIT 10", + }, + { + name: "Test limit less than pageSize - order by ts", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeList, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + OrderBy: []v3.OrderBy{{ColumnName: constants.TIMESTAMP, Order: "desc", Key: constants.TIMESTAMP, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}}, + Limit: 1, + Offset: 0, + PageSize: 5, + }, + }, + want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string from " + + "signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "order by `timestamp` desc LIMIT 1", + }, + { + name: "Test limit greater than pageSize - order by ts", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeList, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "id", Type: v3.AttributeKeyTypeUnspecified, DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Operator: v3.FilterOperatorLessThan, Value: "2TNh4vp2TpiWyLt3SzuadLJF2s4"}, + }}, + OrderBy: []v3.OrderBy{{ColumnName: constants.TIMESTAMP, Order: "desc", Key: constants.TIMESTAMP, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}}, + Limit: 100, + Offset: 10, + PageSize: 10, + }, + }, + want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string from " + + "signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by `timestamp` desc LIMIT 10", + }, + { + name: "Test limit less than pageSize - order by custom", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeList, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "desc", Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + Limit: 1, + Offset: 0, + PageSize: 5, + }, + }, + want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string from " + + "signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "order by attributes_string['method'] desc LIMIT 1 OFFSET 0", + }, + { + name: "Test limit greater than pageSize - order by custom", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeList, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "id", Type: v3.AttributeKeyTypeUnspecified, DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Operator: v3.FilterOperatorLessThan, Value: "2TNh4vp2TpiWyLt3SzuadLJF2s4"}, + }}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "desc", Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + Limit: 100, + Offset: 50, + PageSize: 50, + }, + }, + want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string from " + + "signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND " + + "id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by attributes_string['method'] desc LIMIT 50 OFFSET 50", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := PrepareLogsQuery(tt.args.start, tt.args.end, tt.args.queryType, tt.args.panelType, tt.args.mq, tt.args.options) + if (err != nil) != tt.wantErr { + t.Errorf("PrepareLogsQuery() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("PrepareLogsQuery() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 26824813a8..074af1da6a 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -312,10 +312,10 @@ const ( "CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool," + "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string " LogsSQLSelectV2 = "SELECT " + - "timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body," + - "attributes_string," + - "attributes_number," + - "attributes_bool," + + "timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, " + + "attributes_string, " + + "attributes_number, " + + "attributes_bool, " + "resources_string " TracesExplorerViewSQLSelectWithSubQuery = "WITH subQuery AS (SELECT distinct on (traceID) traceID, durationNano, " + "serviceName, name FROM %s.%s WHERE parentSpanID = '' AND %s %s ORDER BY durationNano DESC " From dc66eb14dee8de2e5bdcdeb8956d366db18a7d03 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 15:43:01 +0530 Subject: [PATCH 04/17] fix: tests fixed --- pkg/query-service/app/logs/v4/query_builder_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go index 69c8a7093e..3d238d2a0c 100644 --- a/pkg/query-service/app/logs/v4/query_builder_test.go +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -342,7 +342,7 @@ func Test_buildLogsTimeSeriesFilterQuery(t *testing.T) { }, }, }, - want: " AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + "AND attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method')", }, { @@ -374,7 +374,7 @@ func Test_buildLogsTimeSeriesFilterQuery(t *testing.T) { Type: v3.AttributeKeyTypeTag, }, }, - want: " AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + "AND mapContains(attributes_string, 'user_name') AND mapContains(attributes_string, 'test')", }, { @@ -422,7 +422,7 @@ func Test_buildLogsTimeSeriesFilterQuery(t *testing.T) { Type: v3.AttributeKeyTypeTag, }, }, - want: " AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + + want: "attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') " + "AND mapContains(attributes_string, 'user_name') AND `attribute_string_method_exists`=true AND mapContains(attributes_string, 'test')", }, } From 2b75ed9092105c2a7c163a68b236c4ec336e0587 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 16:42:51 +0530 Subject: [PATCH 05/17] feat: logs list API, logic update for better perf --- pkg/query-service/app/querier/querier.go | 111 ++++++++++++++++++ pkg/query-service/app/querier/querier_test.go | 45 +++++++ pkg/query-service/app/querier/v2/querier.go | 111 ++++++++++++++++++ .../app/querier/v2/querier_test.go | 46 ++++++++ 4 files changed, 313 insertions(+) diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 86a77da114..359159d28b 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -15,6 +15,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" chErrors "go.signoz.io/signoz/pkg/query-service/errors" + "go.signoz.io/signoz/pkg/query-service/utils" "go.signoz.io/signoz/pkg/query-service/cache" "go.signoz.io/signoz/pkg/query-service/interfaces" @@ -466,7 +467,117 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } +type logsListTsRange struct { + Start int64 + End int64 +} + +const HOUR_NANO = int64(3600000000000) + +func getLogsListTsRanges(start, end int64) []logsListTsRange { + startNano := utils.GetEpochNanoSecs(start) + endNano := utils.GetEpochNanoSecs(end) + result := []logsListTsRange{} + + if endNano-startNano > HOUR_NANO { + bucket := HOUR_NANO + tStartNano := endNano - bucket + + complete := false + for { + result = append(result, logsListTsRange{Start: tStartNano, End: endNano}) + if complete { + break + } + + bucket = bucket * 2 + endNano = tStartNano + tStartNano = tStartNano - bucket + + // break condition + if tStartNano <= startNano { + complete = true + tStartNano = startNano + } + } + } + return result +} + +func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []logsListTsRange) ([]*v3.Result, map[string]error, error) { + res := make([]*v3.Result, 0) + qName := "" + pageSize := uint64(0) + + // se we are considering only one query + for name, v := range params.CompositeQuery.BuilderQueries { + qName = name + pageSize = v.PageSize + } + data := []*v3.Row{} + + for _, v := range tsRanges { + params.Start = v.Start + params.End = v.End + + params.CompositeQuery.BuilderQueries[qName].PageSize = pageSize - uint64(len(data)) + queries, err := q.builder.PrepareQueries(params, keys) + if err != nil { + return nil, nil, err + } + + // this will to run only once + for name, query := range queries { + rowList, err := q.reader.GetListResultV3(ctx, query) + if err != nil { + errs := []error{err} + errQuriesByName := map[string]error{ + name: err, + } + return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) + } + data = append(data, rowList...) + } + + // append a filter to the params + if len(data) > 0 { + params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "id", + IsColumn: true, + DataType: "string", + }, + Operator: v3.FilterOperatorLessThan, + Value: data[len(data)-1].Data["id"], + }) + } + + if uint64(len(data)) >= pageSize { + break + } + } + res = append(res, &v3.Result{ + QueryName: qName, + List: data, + }) + return res, nil, nil +} + func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) ([]*v3.Result, map[string]error, error) { + // List query has support for only one query. + if params.CompositeQuery != nil { + if len(params.CompositeQuery.BuilderQueries) == 1 { + for _, v := range params.CompositeQuery.BuilderQueries { + // only allow of logs queries with timestamp ordering desc + if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { + startEndArr := getLogsListTsRanges(params.Start, params.End) + if len(startEndArr) > 0 { + return q.runLogsListQuery(ctx, params, keys, startEndArr) + } + } + } + } + } queries, err := q.builder.PrepareQueries(params, keys) diff --git a/pkg/query-service/app/querier/querier_test.go b/pkg/query-service/app/querier/querier_test.go index aecb7b27ba..0b6d5710ae 100644 --- a/pkg/query-service/app/querier/querier_test.go +++ b/pkg/query-service/app/querier/querier_test.go @@ -1055,3 +1055,48 @@ func TestQueryRangeValueTypePromQL(t *testing.T) { } } } +func TestLogsListTsRange(t *testing.T) { + startEndData := []struct { + name string + start int64 + end int64 + res []logsListTsRange + }{ + { + name: "testing for less then one hour", + start: 1722262800000000000, // July 29, 2024 7:50:00 PM + end: 1722263800000000000, // July 29, 2024 8:06:40 PM + res: []logsListTsRange{}, + }, + { + name: "testing for more than one hour", + start: 1722255800000000000, // July 29, 2024 5:53:20 PM + end: 1722262800000000000, // July 29, 2024 8:06:40 PM + res: []logsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM + }, + }, + { + name: "testing for 1 day", + start: 1722171576000000000, + end: 1722262800000000000, + res: []logsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM + {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM + {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM + {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM + }, + }, + } + + for _, test := range startEndData { + res := getLogsListTsRanges(test.start, test.end) + for i, v := range res { + if test.res[i].Start != v.Start || test.res[i].End != v.End { + t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) + } + } + } +} diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index d0c3a77d13..ddc7752e58 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -15,6 +15,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" chErrors "go.signoz.io/signoz/pkg/query-service/errors" + "go.signoz.io/signoz/pkg/query-service/utils" "go.signoz.io/signoz/pkg/query-service/cache" "go.signoz.io/signoz/pkg/query-service/interfaces" @@ -475,7 +476,117 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } +type logsListTsRange struct { + Start int64 + End int64 +} + +const HOUR_NANO = int64(3600000000000) + +func getLogsListTsRanges(start, end int64) []logsListTsRange { + startNano := utils.GetEpochNanoSecs(start) + endNano := utils.GetEpochNanoSecs(end) + result := []logsListTsRange{} + + if endNano-startNano > HOUR_NANO { + bucket := HOUR_NANO + tStartNano := endNano - bucket + + complete := false + for { + result = append(result, logsListTsRange{Start: tStartNano, End: endNano}) + if complete { + break + } + + bucket = bucket * 2 + endNano = tStartNano + tStartNano = tStartNano - bucket + + // break condition + if tStartNano <= startNano { + complete = true + tStartNano = startNano + } + } + } + return result +} + +func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []logsListTsRange) ([]*v3.Result, map[string]error, error) { + res := make([]*v3.Result, 0) + qName := "" + pageSize := uint64(0) + + // se we are considering only one query + for name, v := range params.CompositeQuery.BuilderQueries { + qName = name + pageSize = v.PageSize + } + data := []*v3.Row{} + + for _, v := range tsRanges { + params.Start = v.Start + params.End = v.End + + params.CompositeQuery.BuilderQueries[qName].PageSize = pageSize - uint64(len(data)) + queries, err := q.builder.PrepareQueries(params, keys) + if err != nil { + return nil, nil, err + } + + // this will to run only once + for name, query := range queries { + rowList, err := q.reader.GetListResultV3(ctx, query) + if err != nil { + errs := []error{err} + errQuriesByName := map[string]error{ + name: err, + } + return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) + } + data = append(data, rowList...) + } + + // append a filter to the params + if len(data) > 0 { + params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "id", + IsColumn: true, + DataType: "string", + }, + Operator: v3.FilterOperatorLessThan, + Value: data[len(data)-1].Data["id"], + }) + } + + if uint64(len(data)) >= pageSize { + break + } + } + res = append(res, &v3.Result{ + QueryName: qName, + List: data, + }) + return res, nil, nil +} + func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) ([]*v3.Result, map[string]error, error) { + // List query has support for only one query. + if params.CompositeQuery != nil { + if len(params.CompositeQuery.BuilderQueries) == 1 { + for _, v := range params.CompositeQuery.BuilderQueries { + // only allow of logs queries with timestamp ordering desc + if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { + startEndArr := getLogsListTsRanges(params.Start, params.End) + if len(startEndArr) > 0 { + return q.runLogsListQuery(ctx, params, keys, startEndArr) + } + } + } + } + } queries, err := q.builder.PrepareQueries(params, keys) diff --git a/pkg/query-service/app/querier/v2/querier_test.go b/pkg/query-service/app/querier/v2/querier_test.go index 5707e9f70d..8ae1f28c58 100644 --- a/pkg/query-service/app/querier/v2/querier_test.go +++ b/pkg/query-service/app/querier/v2/querier_test.go @@ -1107,3 +1107,49 @@ func TestV2QueryRangeValueTypePromQL(t *testing.T) { } } } + +func TestLogsListTsRange(t *testing.T) { + startEndData := []struct { + name string + start int64 + end int64 + res []logsListTsRange + }{ + { + name: "testing for less then one hour", + start: 1722262800000000000, // July 29, 2024 7:50:00 PM + end: 1722263800000000000, // July 29, 2024 8:06:40 PM + res: []logsListTsRange{}, + }, + { + name: "testing for more than one hour", + start: 1722255800000000000, // July 29, 2024 5:53:20 PM + end: 1722262800000000000, // July 29, 2024 8:06:40 PM + res: []logsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM + }, + }, + { + name: "testing for 1 day", + start: 1722171576000000000, + end: 1722262800000000000, + res: []logsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM + {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM + {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM + {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM + }, + }, + } + + for _, test := range startEndData { + res := getLogsListTsRanges(test.start, test.end) + for i, v := range res { + if test.res[i].Start != v.Start || test.res[i].End != v.End { + t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) + } + } + } +} From 2c96364495f9b77fb5f40d187771b8efdd015194 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 16:47:05 +0530 Subject: [PATCH 06/17] feat: integrate v4 qb --- ee/query-service/app/api/api.go | 4 +- ee/query-service/app/db/reader.go | 2 +- ee/query-service/app/server.go | 12 +- ee/query-service/main.go | 3 + frontend/package.json | 3 +- .../app/clickhouseReader/options.go | 6 + .../app/clickhouseReader/reader.go | 203 ++++++++++++----- pkg/query-service/app/http_handler.go | 33 ++- .../app/logs/v3/query_builder.go | 8 +- .../app/logs/v3/query_builder_test.go | 24 +- pkg/query-service/app/querier/helper.go | 23 +- pkg/query-service/app/querier/querier.go | 29 ++- pkg/query-service/app/querier/v2/helper.go | 22 +- pkg/query-service/app/querier/v2/querier.go | 29 ++- .../app/queryBuilder/query_builder.go | 11 +- .../app/queryBuilder/query_builder_test.go | 212 ++++++++++++++++++ pkg/query-service/app/server.go | 25 ++- pkg/query-service/main.go | 3 + pkg/query-service/rules/manager.go | 63 +++--- pkg/query-service/rules/threshold_rule.go | 9 +- .../rules/threshold_rule_test.go | 4 +- .../tests/integration/test_utils.go | 1 + 22 files changed, 554 insertions(+), 175 deletions(-) diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index 66b462e167..2e2eb8ded5 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -38,7 +38,8 @@ type APIHandlerOptions struct { Cache cache.Cache Gateway *httputil.ReverseProxy // Querier Influx Interval - FluxInterval time.Duration + FluxInterval time.Duration + UseLogsNewSchema bool } type APIHandler struct { @@ -63,6 +64,7 @@ func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) { LogsParsingPipelineController: opts.LogsParsingPipelineController, Cache: opts.Cache, FluxInterval: opts.FluxInterval, + UseLogsNewSchema: opts.UseLogsNewSchema, }) if err != nil { diff --git a/ee/query-service/app/db/reader.go b/ee/query-service/app/db/reader.go index b8326058ec..089f2ff2de 100644 --- a/ee/query-service/app/db/reader.go +++ b/ee/query-service/app/db/reader.go @@ -26,7 +26,7 @@ func NewDataConnector( dialTimeout time.Duration, cluster string, ) *ClickhouseReader { - ch := basechr.NewReader(localDB, promConfigPath, lm, maxIdleConns, maxOpenConns, dialTimeout, cluster) + ch := basechr.NewReader(localDB, promConfigPath, lm, maxIdleConns, maxOpenConns, dialTimeout, cluster, useLogsNewSchema) return &ClickhouseReader{ conn: ch.GetConn(), appdb: localDB, diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index 7fb9317946..ffaea2640e 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -77,6 +77,7 @@ type ServerOptions struct { FluxInterval string Cluster string GatewayUrl string + UseLogsNewSchema bool } // Server runs HTTP api service @@ -176,7 +177,9 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { localDB, reader, serverOptions.DisableRules, - lm) + lm, + serverOptions.UseLogsNewSchema, + ) if err != nil { return nil, err @@ -265,6 +268,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { Cache: c, FluxInterval: fluxInterval, Gateway: gatewayProxy, + UseLogsNewSchema: serverOptions.UseLogsNewSchema, } apiHandler, err := api.NewAPIHandler(apiOpts) @@ -728,7 +732,8 @@ func makeRulesManager( db *sqlx.DB, ch baseint.Reader, disableRules bool, - fm baseint.FeatureLookup) (*baserules.Manager, error) { + fm baseint.FeatureLookup, + useLogsNewSchema bool) (*baserules.Manager, error) { // create engine pqle, err := pqle.FromConfigPath(promConfigPath) @@ -759,7 +764,8 @@ func makeRulesManager( Reader: ch, EvalDelay: baseconst.GetEvalDelay(), - PrepareTaskFunc: rules.PrepareTaskFunc, + PrepareTaskFunc: rules.PrepareTaskFunc, + UseLogsNewSchema: useLogsNewSchema, } // create Manager diff --git a/ee/query-service/main.go b/ee/query-service/main.go index c5a03f4c0f..d9751a5046 100644 --- a/ee/query-service/main.go +++ b/ee/query-service/main.go @@ -87,6 +87,7 @@ func main() { var ruleRepoURL string var cluster string + var useLogsNewSchema bool var cacheConfigPath, fluxInterval string var enableQueryServiceLogOTLPExport bool var preferSpanMetrics bool @@ -96,6 +97,7 @@ func main() { var dialTimeout time.Duration var gatewayUrl string + flag.BoolVar(&useLogsNewSchema, "use-logs-new-schema", false, "force logs_v2 schema for logs") flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.StringVar(&skipTopLvlOpsPath, "skip-top-level-ops", "", "(config file to skip top level operations)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") @@ -134,6 +136,7 @@ func main() { FluxInterval: fluxInterval, Cluster: cluster, GatewayUrl: gatewayUrl, + UseLogsNewSchema: useLogsNewSchema, } // Read the jwt secret key diff --git a/frontend/package.json b/frontend/package.json index 51097f7696..f419c71ce3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -241,5 +241,6 @@ "semver": "7.5.4", "xml2js": "0.5.0", "phin": "^3.7.1" - } + }, + "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e" } diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 538cef33e5..3f564d66a0 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -32,7 +32,9 @@ const ( defaultSpanAttributeKeysTable string = "distributed_span_attributes_keys" defaultLogsDB string = "signoz_logs" defaultLogsTable string = "distributed_logs" + defaultLogsTableV2 string = "distributed_logs_v2" defaultLogsLocalTable string = "logs" + defaultLogsLocalTableV2 string = "logs_v2" defaultLogAttributeKeysTable string = "distributed_logs_attribute_keys" defaultLogResourceKeysTable string = "distributed_logs_resource_keys" defaultLogTagAttributeTable string = "distributed_tag_attributes" @@ -63,7 +65,9 @@ type namespaceConfig struct { TopLevelOperationsTable string LogsDB string LogsTable string + LogsTableV2 string LogsLocalTable string + LogsLocalTableV2 string LogsAttributeKeysTable string LogsResourceKeysTable string LogsTagAttributeTable string @@ -150,7 +154,9 @@ func NewOptions( TopLevelOperationsTable: defaultTopLevelOperationsTable, LogsDB: defaultLogsDB, LogsTable: defaultLogsTable, + LogsTableV2: defaultLogsTableV2, LogsLocalTable: defaultLogsLocalTable, + LogsLocalTableV2: defaultLogsLocalTableV2, LogsAttributeKeysTable: defaultLogAttributeKeysTable, LogsResourceKeysTable: defaultLogResourceKeysTable, LogsTagAttributeTable: defaultLogTagAttributeTable, diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index bac50ca157..4106df5525 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -116,7 +116,9 @@ type ClickHouseReader struct { topLevelOperationsTable string logsDB string logsTable string + logsTableV2 string logsLocalTable string + logsLocalTableV2 string logsAttributeKeys string logsResourceKeys string logsTagAttributeTable string @@ -132,6 +134,8 @@ type ClickHouseReader struct { liveTailRefreshSeconds int cluster string + + UseLogsNewSchema bool } // NewTraceReader returns a TraceReader for the database @@ -143,6 +147,7 @@ func NewReader( maxOpenConns int, dialTimeout time.Duration, cluster string, + useLogsNewSchema bool, ) *ClickHouseReader { datasource := os.Getenv("ClickHouseUrl") @@ -153,7 +158,7 @@ func NewReader( zap.L().Fatal("failed to initialize ClickHouse", zap.Error(err)) } - return NewReaderFromClickhouseConnection(db, options, localDB, configFile, featureFlag, cluster) + return NewReaderFromClickhouseConnection(db, options, localDB, configFile, featureFlag, cluster, useLogsNewSchema) } func NewReaderFromClickhouseConnection( @@ -163,6 +168,7 @@ func NewReaderFromClickhouseConnection( configFile string, featureFlag interfaces.FeatureLookup, cluster string, + useLogsNewSchema bool, ) *ClickHouseReader { alertManager, err := am.New("") if err != nil { @@ -210,7 +216,9 @@ func NewReaderFromClickhouseConnection( topLevelOperationsTable: options.primary.TopLevelOperationsTable, logsDB: options.primary.LogsDB, logsTable: options.primary.LogsTable, + logsTableV2: options.primary.LogsTableV2, logsLocalTable: options.primary.LogsLocalTable, + logsLocalTableV2: options.primary.LogsLocalTableV2, logsAttributeKeys: options.primary.LogsAttributeKeysTable, logsResourceKeys: options.primary.LogsResourceKeysTable, logsTagAttributeTable: options.primary.LogsTagAttributeTable, @@ -219,6 +227,7 @@ func NewReaderFromClickhouseConnection( featureFlags: featureFlag, cluster: cluster, queryProgressTracker: queryprogress.NewQueryProgressTracker(), + UseLogsNewSchema: useLogsNewSchema, } } @@ -3513,7 +3522,11 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe resources = removeUnderscoreDuplicateFields(resources) statements := []model.ShowCreateTableStatement{} - query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsLocalTable) + table := r.logsLocalTable + if r.UseLogsNewSchema { + table = r.logsLocalTableV2 + } + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, table) err = r.db.Select(ctx, &statements, query) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} @@ -3551,68 +3564,132 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda return &model.ApiError{Err: err, Typ: model.ErrorBadData} } - colname := utils.GetClickhouseColumnName(field.Type, field.DataType, field.Name) - // if a field is selected it means that the field needs to be indexed if field.Selected { - keyColName := fmt.Sprintf("%s_%s_key", field.Type, strings.ToLower(field.DataType)) - valueColName := fmt.Sprintf("%s_%s_value", field.Type, strings.ToLower(field.DataType)) - // create materialized column + if r.UseLogsNewSchema { + colname := utils.GetClickhouseColumnNameV2(field.Type, field.DataType, field.Name) + + dataType := strings.ToLower(field.DataType) + if dataType == "int64" || dataType == "float64" { + dataType = "number" + } + attrColName := fmt.Sprintf("%s_%s", field.Type, dataType) + for _, table := range []string{r.logsLocalTableV2, r.logsTableV2} { + q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS `%s` %s DEFAULT %s['%s'] CODEC(ZSTD(1))" + query := fmt.Sprintf(q, + r.logsDB, table, + r.cluster, + colname, field.DataType, + attrColName, + field.Name, + ) + err := r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + + query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS `%s_exists` bool DEFAULT if(mapContains(%s, '%s') != 0, true, false) CODEC(ZSTD(1))", + r.logsDB, table, + r.cluster, + colname, + attrColName, + field.Name, + ) + err = r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + } + + // create the index + if strings.ToLower(field.DataType) == "bool" { + // there is no point in creating index for bool attributes as the cardinality is just 2 + return nil + } - for _, table := range []string{r.logsLocalTable, r.logsTable} { - q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s DEFAULT %s[indexOf(%s, '%s')] CODEC(ZSTD(1))" - query := fmt.Sprintf(q, - r.logsDB, table, + if field.IndexType == "" { + field.IndexType = constants.DefaultLogSkipIndexType + } + if field.IndexGranularity == 0 { + field.IndexGranularity = constants.DefaultLogSkipIndexGranularity + } + query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS `%s_idx` (`%s`) TYPE %s GRANULARITY %d", + r.logsDB, r.logsLocalTableV2, r.cluster, - colname, field.DataType, - valueColName, - keyColName, - field.Name, + colname, + colname, + field.IndexType, + field.IndexGranularity, ) err := r.db.Exec(ctx, query) if err != nil { return &model.ApiError{Err: err, Typ: model.ErrorInternal} } + } else { + // old schema mat columns + + colname := utils.GetClickhouseColumnName(field.Type, field.DataType, field.Name) + + keyColName := fmt.Sprintf("%s_%s_key", field.Type, strings.ToLower(field.DataType)) + valueColName := fmt.Sprintf("%s_%s_value", field.Type, strings.ToLower(field.DataType)) + + // create materialized column + + for _, table := range []string{r.logsLocalTable, r.logsTable} { + q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s DEFAULT %s[indexOf(%s, '%s')] CODEC(ZSTD(1))" + query := fmt.Sprintf(q, + r.logsDB, table, + r.cluster, + colname, field.DataType, + valueColName, + keyColName, + field.Name, + ) + err := r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + + query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists` bool DEFAULT if(indexOf(%s, '%s') != 0, true, false) CODEC(ZSTD(1))", + r.logsDB, table, + r.cluster, + strings.TrimSuffix(colname, "`"), + keyColName, + field.Name, + ) + err = r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + } + + // create the index + if strings.ToLower(field.DataType) == "bool" { + // there is no point in creating index for bool attributes as the cardinality is just 2 + return nil + } - query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists` bool DEFAULT if(indexOf(%s, '%s') != 0, true, false) CODEC(ZSTD(1))", - r.logsDB, table, + if field.IndexType == "" { + field.IndexType = constants.DefaultLogSkipIndexType + } + if field.IndexGranularity == 0 { + field.IndexGranularity = constants.DefaultLogSkipIndexGranularity + } + query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS %s_idx` (%s) TYPE %s GRANULARITY %d", + r.logsDB, r.logsLocalTable, r.cluster, strings.TrimSuffix(colname, "`"), - keyColName, - field.Name, + colname, + field.IndexType, + field.IndexGranularity, ) - err = r.db.Exec(ctx, query) + err := r.db.Exec(ctx, query) if err != nil { return &model.ApiError{Err: err, Typ: model.ErrorInternal} } } - // create the index - if strings.ToLower(field.DataType) == "bool" { - // there is no point in creating index for bool attributes as the cardinality is just 2 - return nil - } - - if field.IndexType == "" { - field.IndexType = constants.DefaultLogSkipIndexType - } - if field.IndexGranularity == 0 { - field.IndexGranularity = constants.DefaultLogSkipIndexGranularity - } - query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS %s_idx` (%s) TYPE %s GRANULARITY %d", - r.logsDB, r.logsLocalTable, - r.cluster, - strings.TrimSuffix(colname, "`"), - colname, - field.IndexType, - field.IndexGranularity, - ) - err := r.db.Exec(ctx, query) - if err != nil { - return &model.ApiError{Err: err, Typ: model.ErrorInternal} - } - } else { // We are not allowing to delete a materialized column // For more details please check https://github.com/SigNoz/signoz/issues/4566 @@ -4145,10 +4222,14 @@ func (r *ClickHouseReader) GetLatestReceivedMetric( return result, nil } -func isColumn(tableStatement, attrType, field, datType string) bool { +func isColumn(useLogsNewSchema bool, tableStatement, attrType, field, datType string) bool { // value of attrType will be `resource` or `tag`, if `tag` change it to `attribute` - name := utils.GetClickhouseColumnName(attrType, datType, field) - + var name string + if useLogsNewSchema { + name = utils.GetClickhouseColumnNameV2(attrType, datType, field) + } else { + name = utils.GetClickhouseColumnName(attrType, datType, field) + } return strings.Contains(tableStatement, fmt.Sprintf("%s ", name)) } @@ -4204,7 +4285,11 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v defer rows.Close() statements := []model.ShowCreateTableStatement{} - query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsLocalTable) + table := r.logsLocalTable + if r.UseLogsNewSchema { + table = r.logsLocalTableV2 + } + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, table) err = r.db.Select(ctx, &statements, query) if err != nil { return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) @@ -4221,7 +4306,7 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v Key: tagKey, DataType: v3.AttributeKeyDataType(dataType), Type: v3.AttributeKeyType(attType), - IsColumn: isColumn(statements[0].Statement, attType, tagKey, dataType), + IsColumn: isColumn(r.UseLogsNewSchema, statements[0].Statement, attType, tagKey, dataType), } response.AttributeKeys = append(response.AttributeKeys, key) } @@ -4258,7 +4343,11 @@ func (r *ClickHouseReader) GetLogAttributeKeys(ctx context.Context, req *v3.Filt defer rows.Close() statements := []model.ShowCreateTableStatement{} - query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsLocalTable) + table := r.logsLocalTable + if r.UseLogsNewSchema { + table = r.logsLocalTableV2 + } + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, table) err = r.db.Select(ctx, &statements, query) if err != nil { return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) @@ -4276,7 +4365,7 @@ func (r *ClickHouseReader) GetLogAttributeKeys(ctx context.Context, req *v3.Filt Key: attributeKey, DataType: v3.AttributeKeyDataType(attributeDataType), Type: v3.AttributeKeyType(tagType), - IsColumn: isColumn(statements[0].Statement, tagType, attributeKey, attributeDataType), + IsColumn: isColumn(r.UseLogsNewSchema, statements[0].Statement, tagType, attributeKey, attributeDataType), } response.AttributeKeys = append(response.AttributeKeys, key) @@ -4310,7 +4399,7 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi } // ignore autocomplete request for body - if req.FilterAttributeKey == "body" { + if req.FilterAttributeKey == "body" || req.FilterAttributeKey == "__attrs" { return &v3.FilterAttributeValueResponse{}, nil } @@ -4332,6 +4421,10 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi } searchText := fmt.Sprintf("%%%s%%", req.SearchText) + table := r.logsLocalTable + if r.UseLogsNewSchema { + table = r.logsLocalTableV2 + } // check if the tagKey is a topLevelColumn if _, ok := constants.StaticFieldsLogsV3[req.FilterAttributeKey]; ok { @@ -4345,10 +4438,10 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi // prepare the query and run if len(req.SearchText) != 0 { - query = fmt.Sprintf("select distinct %s from %s.%s where timestamp >= toInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)*1000000000) and %s ILIKE $1 limit $2", selectKey, r.logsDB, r.logsTable, filterValueColumnWhere) + query = fmt.Sprintf("select distinct %s from %s.%s where timestamp >= toInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)*1000000000) and %s ILIKE $1 limit $2", selectKey, r.logsDB, table, filterValueColumnWhere) rows, err = r.db.Query(ctx, query, searchText, req.Limit) } else { - query = fmt.Sprintf("select distinct %s from %s.%s where timestamp >= toInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)*1000000000) limit $1", selectKey, r.logsDB, r.logsTable) + query = fmt.Sprintf("select distinct %s from %s.%s where timestamp >= toInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)*1000000000) limit $1", selectKey, r.logsDB, table) rows, err = r.db.Query(ctx, query, req.Limit) } } else if len(req.SearchText) != 0 { diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 1210cd4f67..2bc0a2992c 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -29,6 +29,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/integrations" "go.signoz.io/signoz/pkg/query-service/app/logs" logsv3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + logsv4 "go.signoz.io/signoz/pkg/query-service/app/logs/v4" "go.signoz.io/signoz/pkg/query-service/app/metrics" metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" "go.signoz.io/signoz/pkg/query-service/app/preferences" @@ -140,6 +141,9 @@ type APIHandlerOpts struct { // Querier Influx Interval FluxInterval time.Duration + + // Use Logs New schema + UseLogsNewSchema bool } // NewAPIHandler returns an APIHandler @@ -151,19 +155,21 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { } querierOpts := querier.QuerierOptions{ - Reader: opts.Reader, - Cache: opts.Cache, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FluxInterval: opts.FluxInterval, - FeatureLookup: opts.FeatureFlags, + Reader: opts.Reader, + Cache: opts.Cache, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FluxInterval: opts.FluxInterval, + FeatureLookup: opts.FeatureFlags, + UseLogsNewSchema: opts.UseLogsNewSchema, } querierOptsV2 := querierV2.QuerierOptions{ - Reader: opts.Reader, - Cache: opts.Cache, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FluxInterval: opts.FluxInterval, - FeatureLookup: opts.FeatureFlags, + Reader: opts.Reader, + Cache: opts.Cache, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FluxInterval: opts.FluxInterval, + FeatureLookup: opts.FeatureFlags, + UseLogsNewSchema: opts.UseLogsNewSchema, } querier := querier.NewQuerier(querierOpts) @@ -187,10 +193,15 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { querierV2: querierv2, } + logsQueryBuilder := logsv3.PrepareLogsQuery + if opts.UseLogsNewSchema { + logsQueryBuilder = logsv4.PrepareLogsQuery + } + builderOpts := queryBuilder.QueryBuilderOptions{ BuildMetricQuery: metricsv3.PrepareMetricQuery, BuildTraceQuery: tracesV3.PrepareTracesQuery, - BuildLogQuery: logsv3.PrepareLogsQuery, + BuildLogQuery: logsQueryBuilder, } aH.queryBuilder = queryBuilder.NewQueryBuilder(builderOpts, aH.featureFlags) diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index bd64b4d0e6..05bd799712 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -486,12 +486,6 @@ func AddOffsetToQuery(query string, offset uint64) string { return fmt.Sprintf("%s OFFSET %d", query, offset) } -type Options struct { - GraphLimitQtype string - IsLivetailQuery bool - PreferRPM bool -} - func IsOrderByTs(orderBy []v3.OrderBy) bool { if len(orderBy) == 1 && (orderBy[0].Key == constants.TIMESTAMP || orderBy[0].ColumnName == constants.TIMESTAMP) { return true @@ -502,7 +496,7 @@ func IsOrderByTs(orderBy []v3.OrderBy) bool { // PrepareLogsQuery prepares the query for logs // start and end are in epoch millisecond // step is in seconds -func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { +func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) { // adjust the start and end time to the step interval // NOTE: Disabling this as it's creating confusion between charts and actual data diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index db57cb2549..0eb0c202e5 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -1201,7 +1201,7 @@ var testPrepLogsQueryData = []struct { TableName string AggregateOperator v3.AggregateOperator ExpectedQuery string - Options Options + Options v3.LogQBOptions }{ { Name: "Test TS with limit- first", @@ -1223,7 +1223,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by value DESC) LIMIT 10", - Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- first - with order by value", @@ -1246,7 +1246,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by value ASC) LIMIT 10", - Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- first - with order by attribute", @@ -1269,7 +1269,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by `method` ASC) LIMIT 10", - Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- second", @@ -1291,7 +1291,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') AND (`method`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `method`,ts order by value DESC", - Options: Options{GraphLimitQtype: constants.SecondQueryGraphLimit}, + Options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, { Name: "Test TS with limit- second - with order by", @@ -1314,7 +1314,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') AND (`method`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `method`,ts order by `method` ASC", - Options: Options{GraphLimitQtype: constants.SecondQueryGraphLimit}, + Options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, // Live tail { @@ -1334,7 +1334,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND ", - Options: Options{IsLivetailQuery: true}, + Options: v3.LogQBOptions{IsLivetailQuery: true}, }, { Name: "Live Tail Query with contains", @@ -1353,7 +1353,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where attributes_string_value[indexOf(attributes_string_key, 'method')] ILIKE '%GET%' AND ", - Options: Options{IsLivetailQuery: true}, + Options: v3.LogQBOptions{IsLivetailQuery: true}, }, { Name: "Live Tail Query W/O filter", @@ -1369,7 +1369,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where ", - Options: Options{IsLivetailQuery: true}, + Options: v3.LogQBOptions{IsLivetailQuery: true}, }, { Name: "Table query w/o limit", @@ -1385,7 +1385,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) order by value DESC", - Options: Options{}, + Options: v3.LogQBOptions{}, }, { Name: "Table query with limit", @@ -1402,7 +1402,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) order by value DESC LIMIT 10", - Options: Options{}, + Options: v3.LogQBOptions{}, }, { Name: "Ignore offset if order by is timestamp in list queries", @@ -1488,7 +1488,7 @@ var testPrepLogsQueryLimitOffsetData = []struct { TableName string AggregateOperator v3.AggregateOperator ExpectedQuery string - Options Options + Options v3.LogQBOptions }{ { Name: "Test limit less than pageSize - order by ts", diff --git a/pkg/query-service/app/querier/helper.go b/pkg/query-service/app/querier/helper.go index 7c45cc8781..a1c880eac1 100644 --- a/pkg/query-service/app/querier/helper.go +++ b/pkg/query-service/app/querier/helper.go @@ -9,6 +9,7 @@ import ( "time" logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + logsV4 "go.signoz.io/signoz/pkg/query-service/app/logs/v4" metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" "go.signoz.io/signoz/pkg/query-service/cache/status" @@ -19,6 +20,7 @@ import ( ) func prepareLogsQuery(_ context.Context, + useLogsNewSchema bool, start, end int64, builderQuery *v3.BuilderQuery, @@ -27,30 +29,35 @@ func prepareLogsQuery(_ context.Context, ) (string, error) { query := "" + logsQueryBuilder := logsV3.PrepareLogsQuery + if useLogsNewSchema { + logsQueryBuilder = logsV4.PrepareLogsQuery + } + if params == nil || builderQuery == nil { return query, fmt.Errorf("params and builderQuery cannot be nil") } // for ts query with limit replace it as it is already formed if params.CompositeQuery.PanelType == v3.PanelTypeGraph && builderQuery.Limit > 0 && len(builderQuery.GroupBy) > 0 { - limitQuery, err := logsV3.PrepareLogsQuery( + limitQuery, err := logsQueryBuilder( start, end, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - logsV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err } - placeholderQuery, err := logsV3.PrepareLogsQuery( + placeholderQuery, err := logsQueryBuilder( start, end, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - logsV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -59,13 +66,13 @@ func prepareLogsQuery(_ context.Context, return query, err } - query, err := logsV3.PrepareLogsQuery( + query, err := logsQueryBuilder( start, end, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - logsV3.Options{PreferRPM: preferRPM}, + v3.LogQBOptions{PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -102,7 +109,7 @@ func (q *querier) runBuilderQuery( var query string var err error if _, ok := cacheKeys[queryName]; !ok { - query, err = prepareLogsQuery(ctx, start, end, builderQuery, params, preferRPM) + query, err = prepareLogsQuery(ctx, q.UseLogsNewSchema, start, end, builderQuery, params, preferRPM) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: query, Series: nil} return @@ -126,7 +133,7 @@ func (q *querier) runBuilderQuery( missedSeries := make([]*v3.Series, 0) cachedSeries := make([]*v3.Series, 0) for _, miss := range misses { - query, err = prepareLogsQuery(ctx, miss.start, miss.end, builderQuery, params, preferRPM) + query, err = prepareLogsQuery(ctx, q.UseLogsNewSchema, miss.start, miss.end, builderQuery, params, preferRPM) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: query, Series: nil} return diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 359159d28b..af03d15bcf 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -11,6 +11,7 @@ import ( "time" logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + logsV4 "go.signoz.io/signoz/pkg/query-service/app/logs/v4" metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" @@ -52,9 +53,10 @@ type querier struct { testingMode bool queriesExecuted []string // tuple of start and end time in milliseconds - timeRanges [][]int - returnedSeries []*v3.Series - returnedErr error + timeRanges [][]int + returnedSeries []*v3.Series + returnedErr error + UseLogsNewSchema bool } type QuerierOptions struct { @@ -65,12 +67,18 @@ type QuerierOptions struct { FeatureLookup interfaces.FeatureLookup // used for testing - TestingMode bool - ReturnedSeries []*v3.Series - ReturnedErr error + TestingMode bool + ReturnedSeries []*v3.Series + ReturnedErr error + UseLogsNewSchema bool } func NewQuerier(opts QuerierOptions) interfaces.Querier { + logsQueryBuilder := logsV3.PrepareLogsQuery + if opts.UseLogsNewSchema { + logsQueryBuilder = logsV4.PrepareLogsQuery + } + return &querier{ cache: opts.Cache, reader: opts.Reader, @@ -79,14 +87,15 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { builder: queryBuilder.NewQueryBuilder(queryBuilder.QueryBuilderOptions{ BuildTraceQuery: tracesV3.PrepareTracesQuery, - BuildLogQuery: logsV3.PrepareLogsQuery, + BuildLogQuery: logsQueryBuilder, BuildMetricQuery: metricsV3.PrepareMetricQuery, }, opts.FeatureLookup), featureLookUp: opts.FeatureLookup, - testingMode: opts.TestingMode, - returnedSeries: opts.ReturnedSeries, - returnedErr: opts.ReturnedErr, + testingMode: opts.TestingMode, + returnedSeries: opts.ReturnedSeries, + returnedErr: opts.ReturnedErr, + UseLogsNewSchema: opts.UseLogsNewSchema, } } diff --git a/pkg/query-service/app/querier/v2/helper.go b/pkg/query-service/app/querier/v2/helper.go index de9d591f7f..be81e0c024 100644 --- a/pkg/query-service/app/querier/v2/helper.go +++ b/pkg/query-service/app/querier/v2/helper.go @@ -9,6 +9,7 @@ import ( "time" logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + logsV4 "go.signoz.io/signoz/pkg/query-service/app/logs/v4" metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" metricsV4 "go.signoz.io/signoz/pkg/query-service/app/metrics/v4" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" @@ -19,12 +20,17 @@ import ( ) func prepareLogsQuery(_ context.Context, + useLogsNewSchema bool, start, end int64, builderQuery *v3.BuilderQuery, params *v3.QueryRangeParamsV3, preferRPM bool, ) (string, error) { + logsQueryBuilder := logsV3.PrepareLogsQuery + if useLogsNewSchema { + logsQueryBuilder = logsV4.PrepareLogsQuery + } query := "" if params == nil || builderQuery == nil { @@ -33,24 +39,24 @@ func prepareLogsQuery(_ context.Context, // for ts query with limit replace it as it is already formed if params.CompositeQuery.PanelType == v3.PanelTypeGraph && builderQuery.Limit > 0 && len(builderQuery.GroupBy) > 0 { - limitQuery, err := logsV3.PrepareLogsQuery( + limitQuery, err := logsQueryBuilder( start, end, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - logsV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err } - placeholderQuery, err := logsV3.PrepareLogsQuery( + placeholderQuery, err := logsQueryBuilder( start, end, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - logsV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -59,13 +65,13 @@ func prepareLogsQuery(_ context.Context, return query, err } - query, err := logsV3.PrepareLogsQuery( + query, err := logsQueryBuilder( start, end, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - logsV3.Options{PreferRPM: preferRPM}, + v3.LogQBOptions{PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -104,7 +110,7 @@ func (q *querier) runBuilderQuery( var query string var err error if _, ok := cacheKeys[queryName]; !ok { - query, err = prepareLogsQuery(ctx, start, end, builderQuery, params, preferRPM) + query, err = prepareLogsQuery(ctx, q.UseLogsNewSchema, start, end, builderQuery, params, preferRPM) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: query, Series: nil} return @@ -127,7 +133,7 @@ func (q *querier) runBuilderQuery( missedSeries := make([]*v3.Series, 0) cachedSeries := make([]*v3.Series, 0) for _, miss := range misses { - query, err = prepareLogsQuery(ctx, miss.start, miss.end, builderQuery, params, preferRPM) + query, err = prepareLogsQuery(ctx, q.UseLogsNewSchema, miss.start, miss.end, builderQuery, params, preferRPM) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: query, Series: nil} return diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index ddc7752e58..63396076ed 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -11,6 +11,7 @@ import ( "time" logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + logsV4 "go.signoz.io/signoz/pkg/query-service/app/logs/v4" metricsV4 "go.signoz.io/signoz/pkg/query-service/app/metrics/v4" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" @@ -52,9 +53,10 @@ type querier struct { testingMode bool queriesExecuted []string // tuple of start and end time in milliseconds - timeRanges [][]int - returnedSeries []*v3.Series - returnedErr error + timeRanges [][]int + returnedSeries []*v3.Series + returnedErr error + UseLogsNewSchema bool } type QuerierOptions struct { @@ -65,12 +67,18 @@ type QuerierOptions struct { FeatureLookup interfaces.FeatureLookup // used for testing - TestingMode bool - ReturnedSeries []*v3.Series - ReturnedErr error + TestingMode bool + ReturnedSeries []*v3.Series + ReturnedErr error + UseLogsNewSchema bool } func NewQuerier(opts QuerierOptions) interfaces.Querier { + logsQueryBuilder := logsV3.PrepareLogsQuery + if opts.UseLogsNewSchema { + logsQueryBuilder = logsV4.PrepareLogsQuery + } + return &querier{ cache: opts.Cache, reader: opts.Reader, @@ -79,14 +87,15 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { builder: queryBuilder.NewQueryBuilder(queryBuilder.QueryBuilderOptions{ BuildTraceQuery: tracesV3.PrepareTracesQuery, - BuildLogQuery: logsV3.PrepareLogsQuery, + BuildLogQuery: logsQueryBuilder, BuildMetricQuery: metricsV4.PrepareMetricQuery, }, opts.FeatureLookup), featureLookUp: opts.FeatureLookup, - testingMode: opts.TestingMode, - returnedSeries: opts.ReturnedSeries, - returnedErr: opts.ReturnedErr, + testingMode: opts.TestingMode, + returnedSeries: opts.ReturnedSeries, + returnedErr: opts.ReturnedErr, + UseLogsNewSchema: opts.UseLogsNewSchema, } } diff --git a/pkg/query-service/app/queryBuilder/query_builder.go b/pkg/query-service/app/queryBuilder/query_builder.go index 7ddd3c114d..9fe9856798 100644 --- a/pkg/query-service/app/queryBuilder/query_builder.go +++ b/pkg/query-service/app/queryBuilder/query_builder.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/SigNoz/govaluate" - logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" "go.signoz.io/signoz/pkg/query-service/cache" @@ -44,7 +43,7 @@ var SupportedFunctions = []string{ var EvalFuncs = map[string]govaluate.ExpressionFunction{} type prepareTracesQueryFunc func(start, end int64, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey, options tracesV3.Options) (string, error) -type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options logsV3.Options) (string, error) +type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options metricsV3.Options) (string, error) type QueryBuilder struct { @@ -162,7 +161,7 @@ func (qb *QueryBuilder) PrepareLiveTailQuery(params *v3.QueryRangeParamsV3) (str } for queryName, query := range compositeQuery.BuilderQueries { if query.Expression == queryName { - queryStr, err = qb.options.BuildLogQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, logsV3.Options{IsLivetailQuery: true}) + queryStr, err = qb.options.BuildLogQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{IsLivetailQuery: true}) if err != nil { return "", err } @@ -222,18 +221,18 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3, args ...in case v3.DataSourceLogs: // for ts query with limit replace it as it is already formed if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 { - limitQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, logsV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + limitQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } - placeholderQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, logsV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + placeholderQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } query := fmt.Sprintf(placeholderQuery, limitQuery) queries[queryName] = query } else { - queryString, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, logsV3.Options{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) + queryString, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) if err != nil { return nil, err } diff --git a/pkg/query-service/app/queryBuilder/query_builder_test.go b/pkg/query-service/app/queryBuilder/query_builder_test.go index cca8e4a028..38e37a72f3 100644 --- a/pkg/query-service/app/queryBuilder/query_builder_test.go +++ b/pkg/query-service/app/queryBuilder/query_builder_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + logsV4 "go.signoz.io/signoz/pkg/query-service/app/logs/v4" metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/featureManager" @@ -585,6 +586,217 @@ func TestLogsQueryWithFormula(t *testing.T) { } +var testLogsWithFormulaV2 = []struct { + Name string + Query *v3.QueryRangeParamsV3 + ExpectedQuery string +}{ + { + Name: "test formula without dot in filter and group by attribute", + Query: &v3.QueryRangeParamsV3{ + Start: 1702979275000000000, + End: 1702981075000000000, + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + PanelType: v3.PanelTypeGraph, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceLogs, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "key_1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, Value: true, Operator: v3.FilterOperatorEqual}, + }}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, + }, + GroupBy: []v3.AttributeKey{ + {Key: "key_1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, + }, + }, + "B": { + QueryName: "B", + StepInterval: 60, + DataSource: v3.DataSourceLogs, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "key_2", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, Value: true, Operator: v3.FilterOperatorEqual}, + }}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "B", + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, + }, + GroupBy: []v3.AttributeKey{ + {Key: "key_1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, + }, + }, + "C": { + QueryName: "C", + Expression: "A + B", + }, + }, + }, + }, + ExpectedQuery: "SELECT A.`key_1` as `key_1`, A.`ts` as `ts`, A.value + B.value as value FROM " + + "(SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_bool['key_1'] as `key_1`, toFloat64(count(*)) as value from " + + "signoz_logs.distributed_logs_v2 where (timestamp >= 1702979275000000000 AND timestamp <= 1702981075000000000) AND (ts_bucket_start >= 1702977475 AND ts_bucket_start <= 1702981075) " + + "AND attributes_bool['key_1'] = true AND mapContains(attributes_bool, 'key_1') AND mapContains(attributes_bool, 'key_1') group by `key_1`,ts order by value DESC) as A INNER JOIN (SELECT " + + "toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_bool['key_1'] as `key_1`, toFloat64(count(*)) as value " + + "from signoz_logs.distributed_logs_v2 where (timestamp >= 1702979275000000000 AND timestamp <= 1702981075000000000) AND (ts_bucket_start >= 1702977475 AND ts_bucket_start <= 1702981075) " + + "AND attributes_bool['key_2'] = true AND mapContains(attributes_bool, 'key_2') AND mapContains(attributes_bool, 'key_1') group by `key_1`,ts order by value DESC) as B ON A.`key_1` = B.`key_1` AND A.`ts` = B.`ts`", + }, + { + Name: "test formula with dot in filter and group by attribute", + Query: &v3.QueryRangeParamsV3{ + Start: 1702979056000000000, + End: 1702982656000000000, + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + PanelType: v3.PanelTypeTable, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceLogs, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "key1.1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, Value: true, Operator: v3.FilterOperatorEqual}, + }}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, + }, + GroupBy: []v3.AttributeKey{ + {Key: "key1.1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, + }, + }, + "B": { + QueryName: "B", + StepInterval: 60, + DataSource: v3.DataSourceLogs, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "key1.2", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, Value: true, Operator: v3.FilterOperatorEqual}, + }}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "B", + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, + }, + GroupBy: []v3.AttributeKey{ + {Key: "key1.1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, + }, + }, + "C": { + QueryName: "C", + Expression: "A + B", + }, + }, + }, + }, + ExpectedQuery: "SELECT A.`key1.1` as `key1.1`, A.`ts` as `ts`, A.value + B.value as value FROM (SELECT now() as ts, attributes_bool['key1.1'] as `key1.1`, " + + "toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1702979056000000000 AND timestamp <= 1702982656000000000) AND (ts_bucket_start >= 1702977256 AND ts_bucket_start <= 1702982656) " + + "AND attributes_bool['key1.1'] = true AND mapContains(attributes_bool, 'key1.1') AND mapContains(attributes_bool, 'key1.1') group by `key1.1` order by value DESC) as A INNER JOIN (SELECT now() as ts, " + + "attributes_bool['key1.1'] as `key1.1`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1702979056000000000 AND timestamp <= 1702982656000000000) " + + "AND (ts_bucket_start >= 1702977256 AND ts_bucket_start <= 1702982656) AND attributes_bool['key1.2'] = true AND mapContains(attributes_bool, 'key1.2') AND " + + "mapContains(attributes_bool, 'key1.1') group by `key1.1` order by value DESC) as B ON A.`key1.1` = B.`key1.1` AND A.`ts` = B.`ts`", + }, + { + Name: "test formula with dot in filter and group by materialized attribute", + Query: &v3.QueryRangeParamsV3{ + Start: 1702980884000000000, + End: 1702984484000000000, + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + PanelType: v3.PanelTypeGraph, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceLogs, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "key_2", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: true, Operator: v3.FilterOperatorEqual}, + }}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, + }, + GroupBy: []v3.AttributeKey{ + {Key: "key1.1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + }, + "B": { + QueryName: "B", + StepInterval: 60, + DataSource: v3.DataSourceLogs, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "key_1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, Value: true, Operator: v3.FilterOperatorEqual}, + }}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "B", + OrderBy: []v3.OrderBy{ + { + ColumnName: "timestamp", + Order: "desc", + }, + }, + GroupBy: []v3.AttributeKey{ + {Key: "key1.1", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + }, + "C": { + QueryName: "C", + Expression: "A - B", + }, + }, + }, + }, + ExpectedQuery: "SELECT A.`key1.1` as `key1.1`, A.`ts` as `ts`, A.value - B.value as value FROM (SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, " + + "`attribute_bool_key1$$1` as `key1.1`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1702980884000000000 AND timestamp <= 1702984484000000000) AND " + + "(ts_bucket_start >= 1702979084 AND ts_bucket_start <= 1702984484) AND `attribute_bool_key_2` = true AND `attribute_bool_key1$$1_exists`=true group by `key1.1`,ts order by value DESC) as " + + "A INNER JOIN (SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, `attribute_bool_key1$$1` as `key1.1`, toFloat64(count(*)) as value from " + + "signoz_logs.distributed_logs_v2 where (timestamp >= 1702980884000000000 AND timestamp <= 1702984484000000000) AND (ts_bucket_start >= 1702979084 AND ts_bucket_start <= 1702984484) AND " + + "attributes_bool['key_1'] = true AND mapContains(attributes_bool, 'key_1') AND `attribute_bool_key1$$1_exists`=true group by `key1.1`,ts order by value DESC) as B " + + "ON A.`key1.1` = B.`key1.1` AND A.`ts` = B.`ts`", + }, +} + +func TestLogsQueryWithFormulaV2(t *testing.T) { + t.Parallel() + + qbOptions := QueryBuilderOptions{ + BuildLogQuery: logsV4.PrepareLogsQuery, + } + fm := featureManager.StartManager() + qb := NewQueryBuilder(qbOptions, fm) + + for _, test := range testLogsWithFormulaV2 { + t.Run(test.Name, func(t *testing.T) { + queries, err := qb.PrepareQueries(test.Query) + require.NoError(t, err) + require.Equal(t, test.ExpectedQuery, queries["C"]) + }) + } + +} + func TestGenerateCacheKeysMetricsBuilder(t *testing.T) { testCases := []struct { name string diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 77caa9170b..846d39d676 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -66,6 +66,7 @@ type ServerOptions struct { CacheConfigPath string FluxInterval string Cluster string + UseLogsNewSchema bool } // Server runs HTTP, Mux and a grpc server @@ -128,6 +129,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.MaxOpenConns, serverOptions.DialTimeout, serverOptions.Cluster, + serverOptions.UseLogsNewSchema, ) go clickhouseReader.Start(readerReady) reader = clickhouseReader @@ -144,7 +146,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } <-readerReady - rm, err := makeRulesManager(serverOptions.PromConfigPath, constants.GetAlertManagerApiPrefix(), serverOptions.RuleRepoURL, localDB, reader, serverOptions.DisableRules, fm) + rm, err := makeRulesManager(serverOptions.PromConfigPath, constants.GetAlertManagerApiPrefix(), serverOptions.RuleRepoURL, localDB, reader, serverOptions.DisableRules, fm, serverOptions.UseLogsNewSchema) if err != nil { return nil, err } @@ -197,6 +199,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { LogsParsingPipelineController: logParsingPipelineController, Cache: c, FluxInterval: fluxInterval, + UseLogsNewSchema: serverOptions.UseLogsNewSchema, }) if err != nil { return nil, err @@ -713,7 +716,8 @@ func makeRulesManager( db *sqlx.DB, ch interfaces.Reader, disableRules bool, - fm interfaces.FeatureLookup) (*rules.Manager, error) { + fm interfaces.FeatureLookup, + useLogsNewSchema bool) (*rules.Manager, error) { // create engine pqle, err := pqle.FromReader(ch) @@ -735,14 +739,15 @@ func makeRulesManager( PqlEngine: pqle, Ch: ch.GetConn(), }, - RepoURL: ruleRepoURL, - DBConn: db, - Context: context.Background(), - Logger: nil, - DisableRules: disableRules, - FeatureFlags: fm, - Reader: ch, - EvalDelay: constants.GetEvalDelay(), + RepoURL: ruleRepoURL, + DBConn: db, + Context: context.Background(), + Logger: nil, + DisableRules: disableRules, + FeatureFlags: fm, + Reader: ch, + EvalDelay: constants.GetEvalDelay(), + UseLogsNewSchema: useLogsNewSchema, } // create Manager diff --git a/pkg/query-service/main.go b/pkg/query-service/main.go index 3063e07b12..6d59f516b9 100644 --- a/pkg/query-service/main.go +++ b/pkg/query-service/main.go @@ -33,6 +33,7 @@ func main() { // disables rule execution but allows change to the rule definition var disableRules bool + var useLogsNewSchema bool // the url used to build link in the alert messages in slack and other systems var ruleRepoURL, cacheConfigPath, fluxInterval string var cluster string @@ -43,6 +44,7 @@ func main() { var maxOpenConns int var dialTimeout time.Duration + flag.BoolVar(&useLogsNewSchema, "use-logs-new-schema", false, "force logs_v2 schema for logs") flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.StringVar(&skipTopLvlOpsPath, "skip-top-level-ops", "", "(config file to skip top level operations)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") @@ -79,6 +81,7 @@ func main() { CacheConfigPath: cacheConfigPath, FluxInterval: fluxInterval, Cluster: cluster, + UseLogsNewSchema: useLogsNewSchema, } // Read the jwt secret key diff --git a/pkg/query-service/rules/manager.go b/pkg/query-service/rules/manager.go index 5de680c184..02171813cc 100644 --- a/pkg/query-service/rules/manager.go +++ b/pkg/query-service/rules/manager.go @@ -26,14 +26,15 @@ import ( ) type PrepareTaskOptions struct { - Rule *PostableRule - TaskName string - RuleDB RuleDB - Logger *zap.Logger - Reader interfaces.Reader - FF interfaces.FeatureLookup - ManagerOpts *ManagerOptions - NotifyFunc NotifyFunc + Rule *PostableRule + TaskName string + RuleDB RuleDB + Logger *zap.Logger + Reader interfaces.Reader + FF interfaces.FeatureLookup + ManagerOpts *ManagerOptions + NotifyFunc NotifyFunc + UseLogsNewSchema bool } const taskNamesuffix = "webAppEditor" @@ -73,6 +74,8 @@ type ManagerOptions struct { EvalDelay time.Duration + UseLogsNewSchema bool + PrepareTaskFunc func(opts PrepareTaskOptions) (Task, error) } @@ -94,6 +97,8 @@ type Manager struct { featureFlags interfaces.FeatureLookup reader interfaces.Reader + UseLogsNewSchema bool + prepareTaskFunc func(opts PrepareTaskOptions) (Task, error) } @@ -128,7 +133,8 @@ func defaultPrepareTaskFunc(opts PrepareTaskOptions) (Task, error) { ruleId, opts.Rule, ThresholdRuleOpts{ - EvalDelay: opts.ManagerOpts.EvalDelay, + EvalDelay: opts.ManagerOpts.EvalDelay, + UseLogsNewSchema: opts.UseLogsNewSchema, }, opts.FF, opts.Reader, @@ -326,14 +332,15 @@ func (m *Manager) editTask(rule *PostableRule, taskName string) error { zap.L().Debug("editing a rule task", zap.String("name", taskName)) newTask, err := m.prepareTaskFunc(PrepareTaskOptions{ - Rule: rule, - TaskName: taskName, - RuleDB: m.ruleDB, - Logger: m.logger, - Reader: m.reader, - FF: m.featureFlags, - ManagerOpts: m.opts, - NotifyFunc: m.prepareNotifyFunc(), + Rule: rule, + TaskName: taskName, + RuleDB: m.ruleDB, + Logger: m.logger, + Reader: m.reader, + FF: m.featureFlags, + ManagerOpts: m.opts, + NotifyFunc: m.prepareNotifyFunc(), + UseLogsNewSchema: m.opts.UseLogsNewSchema, }) if err != nil { @@ -445,14 +452,15 @@ func (m *Manager) addTask(rule *PostableRule, taskName string) error { zap.L().Debug("adding a new rule task", zap.String("name", taskName)) newTask, err := m.prepareTaskFunc(PrepareTaskOptions{ - Rule: rule, - TaskName: taskName, - RuleDB: m.ruleDB, - Logger: m.logger, - Reader: m.reader, - FF: m.featureFlags, - ManagerOpts: m.opts, - NotifyFunc: m.prepareNotifyFunc(), + Rule: rule, + TaskName: taskName, + RuleDB: m.ruleDB, + Logger: m.logger, + Reader: m.reader, + FF: m.featureFlags, + ManagerOpts: m.opts, + NotifyFunc: m.prepareNotifyFunc(), + UseLogsNewSchema: m.opts.UseLogsNewSchema, }) for _, r := range newTask.Rules() { @@ -794,8 +802,9 @@ func (m *Manager) TestNotification(ctx context.Context, ruleStr string) (int, *m alertname, parsedRule, ThresholdRuleOpts{ - SendUnmatched: true, - SendAlways: true, + SendUnmatched: true, + SendAlways: true, + UseLogsNewSchema: m.UseLogsNewSchema, }, m.featureFlags, m.reader, diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index e657af9288..0034b6d41e 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -98,8 +98,9 @@ type ThresholdRule struct { // querierV2 is used for alerts created after the introduction of new metrics query builder querierV2 interfaces.Querier - reader interfaces.Reader - evalDelay time.Duration + reader interfaces.Reader + evalDelay time.Duration + UseLogsNewSchema bool } type ThresholdRuleOpts struct { @@ -116,7 +117,8 @@ type ThresholdRuleOpts struct { // before evaluating the rule. This is useful in scenarios // where data might not be available in the system immediately // after the timestamp. - EvalDelay time.Duration + EvalDelay time.Duration + UseLogsNewSchema bool } func NewThresholdRule( @@ -151,6 +153,7 @@ func NewThresholdRule( version: p.Version, temporalityMap: make(map[string]map[v3.Temporality]bool), evalDelay: opts.EvalDelay, + UseLogsNewSchema: opts.UseLogsNewSchema, } if int64(t.evalWindow) == 0 { diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index 6cfeac83d9..86ed3aef2e 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -1135,7 +1135,7 @@ func TestThresholdRuleUnitCombinations(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "") + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) rule, err := NewThresholdRule("69", &postableRule, ThresholdRuleOpts{}, fm, reader) rule.temporalityMap = map[string]map[v3.Temporality]bool{ @@ -1238,7 +1238,7 @@ func TestThresholdRuleNoData(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "") + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) rule, err := NewThresholdRule("69", &postableRule, ThresholdRuleOpts{}, fm, reader) rule.temporalityMap = map[string]map[v3.Temporality]bool{ diff --git a/pkg/query-service/tests/integration/test_utils.go b/pkg/query-service/tests/integration/test_utils.go index 65140e5fc8..d060433dba 100644 --- a/pkg/query-service/tests/integration/test_utils.go +++ b/pkg/query-service/tests/integration/test_utils.go @@ -45,6 +45,7 @@ func NewMockClickhouseReader( "", featureFlags, "", + true, ) return reader, mockDB From 14c0c62bfd0f4cb793420e25e76b45f0de91b964 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 16:51:10 +0530 Subject: [PATCH 07/17] fix: pass flag --- ee/query-service/app/db/reader.go | 1 + ee/query-service/app/server.go | 1 + frontend/package.json | 3 +-- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ee/query-service/app/db/reader.go b/ee/query-service/app/db/reader.go index 089f2ff2de..fcab1cb991 100644 --- a/ee/query-service/app/db/reader.go +++ b/ee/query-service/app/db/reader.go @@ -25,6 +25,7 @@ func NewDataConnector( maxOpenConns int, dialTimeout time.Duration, cluster string, + useLogsNewSchema bool, ) *ClickhouseReader { ch := basechr.NewReader(localDB, promConfigPath, lm, maxIdleConns, maxOpenConns, dialTimeout, cluster, useLogsNewSchema) return &ClickhouseReader{ diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index ffaea2640e..cb802f1102 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -155,6 +155,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.MaxOpenConns, serverOptions.DialTimeout, serverOptions.Cluster, + serverOptions.UseLogsNewSchema, ) go qb.Start(readerReady) reader = qb diff --git a/frontend/package.json b/frontend/package.json index f419c71ce3..51097f7696 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -241,6 +241,5 @@ "semver": "7.5.4", "xml2js": "0.5.0", "phin": "^3.7.1" - }, - "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e" + } } From ff0ab8be63f89d0e5b6a93e199d67ef007ce87dc Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 17:44:04 +0530 Subject: [PATCH 08/17] fix: update select for table panel --- pkg/query-service/app/logs/v4/query_builder.go | 3 ++- pkg/query-service/app/logs/v4/query_builder_test.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index 53909542b5..3d0649a4e4 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -61,6 +61,7 @@ func getClickhouseKey(key v3.AttributeKey) string { } // materialized column created from query + // https://github.com/SigNoz/signoz/pull/4775 return "`" + utils.GetClickhouseColumnNameV2(string(key.Type), string(key.DataType), key.Key) + "`" } @@ -417,7 +418,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build queryTmplPrefix = "SELECT" } else if panelType == v3.PanelTypeTable { queryTmplPrefix = - "SELECT now() as ts," + "SELECT" // step or aggregate interval is whole time period in case of table panel step = (utils.GetEpochNanoSecs(end) - utils.GetEpochNanoSecs(start)) / NANOSECOND } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go index 3d238d2a0c..eb1506d4d5 100644 --- a/pkg/query-service/app/logs/v4/query_builder_test.go +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -679,7 +679,7 @@ func Test_buildLogsQuery(t *testing.T) { }, }, }, - want: "SELECT now() as ts, attributes_string['user_name'] as `user_name`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 " + + want: "SELECT attributes_string['user_name'] as `user_name`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + "AND attributes_string['service.name'] = 'test' AND mapContains(attributes_string, 'service.name') AND mapContains(attributes_string, 'user_name') " + "group by `user_name` order by `user_name` desc", @@ -839,7 +839,7 @@ func TestPrepareLogsQuery(t *testing.T) { }, }, }, - want: "SELECT now() as ts, attributes_string['name'] as `name`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 " + + want: "SELECT attributes_string['name'] as `name`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + "AND lower(body) like lower('%requestor_list%') AND lower(body) like lower('%index_service%') AND " + "has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service') AND mapContains(attributes_string, 'name') " + @@ -955,7 +955,7 @@ func TestPrepareLogsQuery(t *testing.T) { Limit: 10, }, }, - want: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) order by value DESC LIMIT 10", + want: "SELECT toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) order by value DESC LIMIT 10", }, { name: "Test limit less than pageSize - order by ts", From db562044281e33581c556f1e4aa7feee59fa0532 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 17:57:18 +0530 Subject: [PATCH 09/17] fix: tests updated with better examples of limit and group by --- .../app/logs/v4/query_builder_test.go | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go index eb1506d4d5..80fe0e8839 100644 --- a/pkg/query-service/app/logs/v4/query_builder_test.go +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -833,17 +833,18 @@ func TestPrepareLogsQuery(t *testing.T) { }, GroupBy: []v3.AttributeKey{ {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, }, OrderBy: []v3.OrderBy{ {ColumnName: "name", Order: "DESC"}, }, }, }, - want: "SELECT attributes_string['name'] as `name`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 " + - "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + - "AND lower(body) like lower('%requestor_list%') AND lower(body) like lower('%index_service%') AND " + - "has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service') AND mapContains(attributes_string, 'name') " + - "group by `name` order by `name` DESC", + want: "SELECT attributes_string['name'] as `name`, resources_string['host'] as `host`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where " + + "(timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND lower(body) like lower('%requestor_list%') " + + "AND lower(body) like lower('%index_service%') AND has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service') AND mapContains(attributes_string, 'name') AND " + + "(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) AND " + + "( (simpleJSONHas(labels, 'host') AND labels like '%host%') ))) group by `name`,`host` order by `name` DESC", }, { name: "Test TS with limit- first", @@ -860,16 +861,19 @@ func TestPrepareLogsQuery(t *testing.T) { Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + {Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "app", Operator: "="}, }, }, Limit: 10, - GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + GroupBy: []v3.AttributeKey{{Key: "user", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, }, options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, - want: "SELECT `method` from (SELECT attributes_string['method'] as `method`, toFloat64(count(distinct(attributes_string['name']))) as value from signoz_logs.distributed_logs_v2 " + + want: "SELECT `user` from (SELECT attributes_string['user'] as `user`, toFloat64(count(distinct(attributes_string['name']))) as value from signoz_logs.distributed_logs_v2 " + "where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['method'] = 'GET' " + - "AND mapContains(attributes_string, 'method') AND mapContains(attributes_string, 'method') AND mapContains(attributes_string, 'name') group by `method` order by value DESC) LIMIT 10", + "AND mapContains(attributes_string, 'method') AND mapContains(attributes_string, 'user') AND mapContains(attributes_string, 'name') AND (resource_fingerprint GLOBAL IN " + + "(SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) AND simpleJSONExtractString(labels, 'service.name') = 'app' " + + "AND labels like '%service.name%app%')) group by `user` order by value DESC) LIMIT 10", }, { name: "Test TS with limit- second", @@ -886,17 +890,19 @@ func TestPrepareLogsQuery(t *testing.T) { Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + {Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "app", Operator: "="}, }, }, - GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + GroupBy: []v3.AttributeKey{{Key: "user", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, Limit: 2, }, options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, - want: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string['method'] as `method`, toFloat64(count(distinct(attributes_string['name']))) as value " + - "from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + - "AND attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND mapContains(attributes_string, 'method') " + - "AND mapContains(attributes_string, 'name') AND (`method`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `method`,ts order by value DESC", + want: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string['user'] as `user`, toFloat64(count(distinct(attributes_string['name']))) as value " + + "from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND " + + "attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND mapContains(attributes_string, 'user') AND mapContains(attributes_string, 'name') AND " + + "(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) AND " + + "simpleJSONExtractString(labels, 'service.name') = 'app' AND labels like '%service.name%app%')) AND (`user`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `user`,ts order by value DESC", }, { name: "Live Tail Query", From 34c2c7b236db8a3b7e5cc8188b5419756877601d Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 19:18:59 +0530 Subject: [PATCH 10/17] fix: resource filter support in live tail --- .../app/logs/v4/query_builder.go | 13 +++++++++- .../app/logs/v4/query_builder_test.go | 24 +++++++++++++++++++ .../app/logs/v4/resource_query_builder.go | 13 ++++++---- .../logs/v4/resource_query_builder_test.go | 2 +- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index 3d0649a4e4..b96c5b9113 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -354,7 +354,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build } // build the where clause for resource table - resourceSubQuery, err := buildResourceSubQuery(bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute) + resourceSubQuery, err := buildResourceSubQuery(bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) if err != nil { return "", err } @@ -444,6 +444,17 @@ func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) { return "", err } + // no values for bucket start and end + resourceSubQuery, err := buildResourceSubQuery(0, 0, mq.Filters, mq.GroupBy, mq.AggregateAttribute, true) + if err != nil { + return "", err + } + // join both the filter clauses + if resourceSubQuery != "" { + filterSubQuery = filterSubQuery + " AND (resource_fingerprint GLOBAL IN " + resourceSubQuery + } + + // the reader will add the timestamp and id filters switch mq.AggregateOperator { case v3.AggregateOperatorNoOp: query := constants.LogsSQLSelectV2 + "from signoz_logs." + DISTRIBUTED_LOGS_V2 + " where " diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go index 80fe0e8839..7bc831437c 100644 --- a/pkg/query-service/app/logs/v4/query_builder_test.go +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -926,6 +926,30 @@ func TestPrepareLogsQuery(t *testing.T) { want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string " + "from signoz_logs.distributed_logs_v2 where attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND ", }, + { + name: "Live Tail Query with resource attribute", + args: args{ + start: 1680066360726, + end: 1680066458000, + queryType: v3.QueryTypeBuilder, + panelType: v3.PanelTypeList, + mq: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + {Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "app", Operator: "contains"}, + }, + }, + }, + options: v3.LogQBOptions{IsLivetailQuery: true}, + }, + want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string from " + + "signoz_logs.distributed_logs_v2 where attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND " + + "(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE simpleJSONExtractString(labels, 'service.name') LIKE '%app%' AND labels like '%service.name%app%' AND ", + }, { name: "Live Tail Query W/O filter", args: args{ diff --git a/pkg/query-service/app/logs/v4/resource_query_builder.go b/pkg/query-service/app/logs/v4/resource_query_builder.go index 004c9269fb..12d6c1a36a 100644 --- a/pkg/query-service/app/logs/v4/resource_query_builder.go +++ b/pkg/query-service/app/logs/v4/resource_query_builder.go @@ -164,7 +164,7 @@ func buildResourceFiltersFromAggregateAttribute(aggregateAttribute v3.AttributeK return "" } -func buildResourceSubQuery(bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey) (string, error) { +func buildResourceSubQuery(bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey, isLiveTail bool) (string, error) { // BUILD THE WHERE CLAUSE var conditions []string @@ -193,9 +193,14 @@ func buildResourceSubQuery(bucketStart, bucketEnd int64, fs *v3.FilterSet, group conditionStr := strings.Join(conditions, " AND ") // BUILD THE FINAL QUERY - query := fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE (seen_at_ts_bucket_start >= %d) AND (seen_at_ts_bucket_start <= %d) AND ", DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd) - - query = "(" + query + conditionStr + ")" + var query string + if isLiveTail { + query = fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE ", DISTRIBUTED_LOGS_V2_RESOURCE) + query = "(" + query + conditionStr + } else { + query = fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE (seen_at_ts_bucket_start >= %d) AND (seen_at_ts_bucket_start <= %d) AND ", DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd) + query = "(" + query + conditionStr + ")" + } return query, nil } diff --git a/pkg/query-service/app/logs/v4/resource_query_builder_test.go b/pkg/query-service/app/logs/v4/resource_query_builder_test.go index 1616c29e08..130fd9e98c 100644 --- a/pkg/query-service/app/logs/v4/resource_query_builder_test.go +++ b/pkg/query-service/app/logs/v4/resource_query_builder_test.go @@ -469,7 +469,7 @@ func Test_buildResourceSubQuery(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := buildResourceSubQuery(tt.args.bucketStart, tt.args.bucketEnd, tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute) + got, err := buildResourceSubQuery(tt.args.bucketStart, tt.args.bucketEnd, tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute, false) if (err != nil) != tt.wantErr { t.Errorf("buildResourceSubQuery() error = %v, wantErr %v", err, tt.wantErr) return From fc08bd93d043ab021a834452420fd111040db0bf Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 10 Sep 2024 19:25:08 +0530 Subject: [PATCH 11/17] fix: v4 livetail api --- .../app/clickhouseReader/options.go | 3 + .../app/clickhouseReader/reader.go | 53 ++++++++++++++ pkg/query-service/app/http_handler.go | 69 +++++++++++++------ .../app/queryBuilder/query_builder_test.go | 4 +- pkg/query-service/interfaces/interface.go | 1 + pkg/query-service/model/response.go | 15 ++++ pkg/query-service/model/v3/v3.go | 7 ++ 7 files changed, 130 insertions(+), 22 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 3f564d66a0..bd6356ec4a 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -33,6 +33,7 @@ const ( defaultLogsDB string = "signoz_logs" defaultLogsTable string = "distributed_logs" defaultLogsTableV2 string = "distributed_logs_v2" + defaultLogsResourceV2 string = "distributed_logs_v2_resource" defaultLogsLocalTable string = "logs" defaultLogsLocalTableV2 string = "logs_v2" defaultLogAttributeKeysTable string = "distributed_logs_attribute_keys" @@ -68,6 +69,7 @@ type namespaceConfig struct { LogsTableV2 string LogsLocalTable string LogsLocalTableV2 string + LogsResourceV2 string LogsAttributeKeysTable string LogsResourceKeysTable string LogsTagAttributeTable string @@ -155,6 +157,7 @@ func NewOptions( LogsDB: defaultLogsDB, LogsTable: defaultLogsTable, LogsTableV2: defaultLogsTableV2, + LogsResourceV2: defaultLogsResourceV2, LogsLocalTable: defaultLogsLocalTable, LogsLocalTableV2: defaultLogsLocalTableV2, LogsAttributeKeysTable: defaultLogAttributeKeysTable, diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 670e6fb248..22f72a679f 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -89,6 +89,7 @@ const ( maxProgressiveSteps = 4 charset = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + NANOSECOND = 1000000000 ) var ( @@ -5350,6 +5351,58 @@ func (r *ClickHouseReader) GetSpanAttributeKeys(ctx context.Context) (map[string return response, nil } +func (r *ClickHouseReader) LiveTailLogsV4(ctx context.Context, query string, timestampStart uint64, idStart string, client *v3.LogsLiveTailClientV2) { + if timestampStart == 0 { + timestampStart = uint64(time.Now().UnixNano()) + } else { + timestampStart = uint64(utils.GetEpochNanoSecs(int64(timestampStart))) + } + + ticker := time.NewTicker(time.Duration(r.liveTailRefreshSeconds) * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + done := true + client.Done <- &done + zap.L().Debug("closing go routine : " + client.Name) + return + case <-ticker.C: + // get the new 100 logs as anything more older won't make sense + var tmpQuery string + bucketStart := (timestampStart / NANOSECOND) - 1800 + + // we have to form the query differently if the resource filters are used + if strings.Contains(query, defaultLogsResourceV2) { + tmpQuery = fmt.Sprintf("seen_at_ts_bucket_start >=%d)) AND ts_bucket_start >=%d AND timestamp >=%d", bucketStart, bucketStart, timestampStart) + } else { + tmpQuery = fmt.Sprintf("ts_bucket_start >=%d AND timestamp >=%d", bucketStart, timestampStart) + } + if idStart != "" { + tmpQuery = fmt.Sprintf("%s AND id > '%s'", tmpQuery, idStart) + } + // the reason we are doing desc is that we need the latest logs first + tmpQuery = query + tmpQuery + " order by timestamp desc, id desc limit 100" + + // using the old structure since we can directly read it to the struct as use it. + response := []model.SignozLogV2{} + err := r.db.Select(ctx, &response, tmpQuery) + if err != nil { + zap.L().Error("Error while getting logs", zap.Error(err)) + client.Error <- err + return + } + for i := len(response) - 1; i >= 0; i-- { + client.Logs <- &response[i] + if i == 0 { + timestampStart = response[i].Timestamp + idStart = response[i].ID + } + } + } + } +} + func (r *ClickHouseReader) LiveTailLogsV3(ctx context.Context, query string, timestampStart uint64, idStart string, client *v3.LogsLiveTailClient) { if timestampStart == 0 { timestampStart = uint64(time.Now().UnixNano()) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 8b5bb46aea..20a1b90455 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -106,6 +106,8 @@ type APIHandler struct { // Websocket connection upgrader Upgrader *websocket.Upgrader + + UseLogsNewSchema bool } type APIHandlerOpts struct { @@ -191,6 +193,7 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { LogsParsingPipelineController: opts.LogsParsingPipelineController, querier: querier, querierV2: querierv2, + UseLogsNewSchema: opts.UseLogsNewSchema, } logsQueryBuilder := logsv3.PrepareLogsQuery @@ -3971,10 +3974,6 @@ func (aH *APIHandler) liveTailLogs(w http.ResponseWriter, r *http.Request) { return } - // create the client - client := &v3.LogsLiveTailClient{Name: r.RemoteAddr, Logs: make(chan *model.SignozLog, 1000), Done: make(chan *bool), Error: make(chan error)} - go aH.reader.LiveTailLogsV3(r.Context(), queryString, uint64(queryRangeParams.Start), "", client) - w.Header().Set("Connection", "keep-alive") w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") @@ -3987,26 +3986,56 @@ func (aH *APIHandler) liveTailLogs(w http.ResponseWriter, r *http.Request) { RespondError(w, &err, "streaming is not supported") return } + // flush the headers flusher.Flush() - for { - select { - case log := <-client.Logs: - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.Encode(log) - fmt.Fprintf(w, "data: %v\n\n", buf.String()) - flusher.Flush() - case <-client.Done: - zap.L().Debug("done!") - return - case err := <-client.Error: - zap.L().Error("error occurred", zap.Error(err)) - fmt.Fprintf(w, "event: error\ndata: %v\n\n", err.Error()) - flusher.Flush() - return + + // create the client + if aH.UseLogsNewSchema { + // create the client + client := &v3.LogsLiveTailClientV2{Name: r.RemoteAddr, Logs: make(chan *model.SignozLogV2, 1000), Done: make(chan *bool), Error: make(chan error)} + go aH.reader.LiveTailLogsV4(r.Context(), queryString, uint64(queryRangeParams.Start), "", client) + for { + select { + case log := <-client.Logs: + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.Encode(log) + fmt.Fprintf(w, "data: %v\n\n", buf.String()) + flusher.Flush() + case <-client.Done: + zap.L().Debug("done!") + return + case err := <-client.Error: + zap.L().Error("error occurred", zap.Error(err)) + fmt.Fprintf(w, "event: error\ndata: %v\n\n", err.Error()) + flusher.Flush() + return + } + } + } else { + client := &v3.LogsLiveTailClient{Name: r.RemoteAddr, Logs: make(chan *model.SignozLog, 1000), Done: make(chan *bool), Error: make(chan error)} + go aH.reader.LiveTailLogsV3(r.Context(), queryString, uint64(queryRangeParams.Start), "", client) + for { + select { + case log := <-client.Logs: + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.Encode(log) + fmt.Fprintf(w, "data: %v\n\n", buf.String()) + flusher.Flush() + case <-client.Done: + zap.L().Debug("done!") + return + case err := <-client.Error: + zap.L().Error("error occurred", zap.Error(err)) + fmt.Fprintf(w, "event: error\ndata: %v\n\n", err.Error()) + flusher.Flush() + return + } } } + } func (aH *APIHandler) getMetricMetadata(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/query-service/app/queryBuilder/query_builder_test.go b/pkg/query-service/app/queryBuilder/query_builder_test.go index 38e37a72f3..52af7af780 100644 --- a/pkg/query-service/app/queryBuilder/query_builder_test.go +++ b/pkg/query-service/app/queryBuilder/query_builder_test.go @@ -707,9 +707,9 @@ var testLogsWithFormulaV2 = []struct { }, }, }, - ExpectedQuery: "SELECT A.`key1.1` as `key1.1`, A.`ts` as `ts`, A.value + B.value as value FROM (SELECT now() as ts, attributes_bool['key1.1'] as `key1.1`, " + + ExpectedQuery: "SELECT A.`key1.1` as `key1.1`, A.`ts` as `ts`, A.value + B.value as value FROM (SELECT attributes_bool['key1.1'] as `key1.1`, " + "toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1702979056000000000 AND timestamp <= 1702982656000000000) AND (ts_bucket_start >= 1702977256 AND ts_bucket_start <= 1702982656) " + - "AND attributes_bool['key1.1'] = true AND mapContains(attributes_bool, 'key1.1') AND mapContains(attributes_bool, 'key1.1') group by `key1.1` order by value DESC) as A INNER JOIN (SELECT now() as ts, " + + "AND attributes_bool['key1.1'] = true AND mapContains(attributes_bool, 'key1.1') AND mapContains(attributes_bool, 'key1.1') group by `key1.1` order by value DESC) as A INNER JOIN (SELECT " + "attributes_bool['key1.1'] as `key1.1`, toFloat64(count(*)) as value from signoz_logs.distributed_logs_v2 where (timestamp >= 1702979056000000000 AND timestamp <= 1702982656000000000) " + "AND (ts_bucket_start >= 1702977256 AND ts_bucket_start <= 1702982656) AND attributes_bool['key1.2'] = true AND mapContains(attributes_bool, 'key1.2') AND " + "mapContains(attributes_bool, 'key1.1') group by `key1.1` order by value DESC) as B ON A.`key1.1` = B.`key1.1` AND A.`ts` = B.`ts`", diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index cfb4f9159e..ec6bb56494 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -71,6 +71,7 @@ type Reader interface { GetTimeSeriesResultV3(ctx context.Context, query string) ([]*v3.Series, error) GetListResultV3(ctx context.Context, query string) ([]*v3.Row, error) LiveTailLogsV3(ctx context.Context, query string, timestampStart uint64, idStart string, client *v3.LogsLiveTailClient) + LiveTailLogsV4(ctx context.Context, query string, timestampStart uint64, idStart string, client *v3.LogsLiveTailClientV2) GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) GetSavedViewsInfo(ctx context.Context) (*model.SavedViewsInfo, error) diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 83df872175..5144dade47 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -572,6 +572,21 @@ type SignozLog struct { Attributes_bool map[string]bool `json:"attributes_bool" ch:"attributes_bool"` } +type SignozLogV2 struct { + Timestamp uint64 `json:"timestamp" ch:"timestamp"` + ID string `json:"id" ch:"id"` + TraceID string `json:"trace_id" ch:"trace_id"` + SpanID string `json:"span_id" ch:"span_id"` + TraceFlags uint32 `json:"trace_flags" ch:"trace_flags"` + SeverityText string `json:"severity_text" ch:"severity_text"` + SeverityNumber uint8 `json:"severity_number" ch:"severity_number"` + Body string `json:"body" ch:"body"` + Resources_string map[string]string `json:"resources_string" ch:"resources_string"` + Attributes_string map[string]string `json:"attributes_string" ch:"attributes_string"` + Attributes_number map[string]float64 `json:"attributes_float" ch:"attributes_number"` + Attributes_bool map[string]bool `json:"attributes_bool" ch:"attributes_bool"` +} + type LogsTailClient struct { Name string Logs chan *SignozLog diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index c21d47229c..4af5d36ae4 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -1035,6 +1035,13 @@ type Result struct { Table *Table `json:"table,omitempty"` } +type LogsLiveTailClientV2 struct { + Name string + Logs chan *model.SignozLogV2 + Done chan *bool + Error chan error +} + type LogsLiveTailClient struct { Name string Logs chan *model.SignozLog From 4574f76f6cf5a26b2b9e2604e3a1bb1e2cc37205 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 11 Sep 2024 16:20:30 +0530 Subject: [PATCH 12/17] fix: changes for casting pointer value --- pkg/query-service/utils/format.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/query-service/utils/format.go b/pkg/query-service/utils/format.go index e9b7a0b7e3..f09f4dffd6 100644 --- a/pkg/query-service/utils/format.go +++ b/pkg/query-service/utils/format.go @@ -14,6 +14,8 @@ import ( // ValidateAndCastValue validates and casts the value of a key to the corresponding data type of the key func ValidateAndCastValue(v interface{}, dataType v3.AttributeKeyDataType) (interface{}, error) { + // get the actual value if it's a pointer + v = getPointerValue(v) switch dataType { case v3.AttributeKeyDataTypeString: switch x := v.(type) { From 8fd3ba5c588a5b575934b92e5b1a2684941fc926 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 11 Sep 2024 17:14:00 +0530 Subject: [PATCH 13/17] fix: reader options updated --- .../app/clickhouseReader/options.go | 24 ++++++++++++------- .../app/clickhouseReader/reader.go | 19 ++++++++++----- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index bd6356ec4a..695bef8570 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -32,10 +32,7 @@ const ( defaultSpanAttributeKeysTable string = "distributed_span_attributes_keys" defaultLogsDB string = "signoz_logs" defaultLogsTable string = "distributed_logs" - defaultLogsTableV2 string = "distributed_logs_v2" - defaultLogsResourceV2 string = "distributed_logs_v2_resource" defaultLogsLocalTable string = "logs" - defaultLogsLocalTableV2 string = "logs_v2" defaultLogAttributeKeysTable string = "distributed_logs_attribute_keys" defaultLogResourceKeysTable string = "distributed_logs_resource_keys" defaultLogTagAttributeTable string = "distributed_tag_attributes" @@ -43,6 +40,11 @@ const ( defaultWriteBatchDelay time.Duration = 5 * time.Second defaultWriteBatchSize int = 10000 defaultEncoding Encoding = EncodingJSON + + defaultLogsLocalTableV2 string = "logs_v2" + defaultLogsTableV2 string = "distributed_logs_v2" + defaultLogsResourceLocalTableV2 string = "logs_v2_resource" + defaultLogsResourceTableV2 string = "distributed_logs_v2_resource" ) // NamespaceConfig is Clickhouse's internal configuration data @@ -66,10 +68,7 @@ type namespaceConfig struct { TopLevelOperationsTable string LogsDB string LogsTable string - LogsTableV2 string LogsLocalTable string - LogsLocalTableV2 string - LogsResourceV2 string LogsAttributeKeysTable string LogsResourceKeysTable string LogsTagAttributeTable string @@ -78,6 +77,11 @@ type namespaceConfig struct { WriteBatchSize int Encoding Encoding Connector Connector + + LogsLocalTableV2 string + LogsTableV2 string + LogsResourceLocalTableV2 string + LogsResourceTableV2 string } // Connecto defines how to connect to the database @@ -156,10 +160,7 @@ func NewOptions( TopLevelOperationsTable: defaultTopLevelOperationsTable, LogsDB: defaultLogsDB, LogsTable: defaultLogsTable, - LogsTableV2: defaultLogsTableV2, - LogsResourceV2: defaultLogsResourceV2, LogsLocalTable: defaultLogsLocalTable, - LogsLocalTableV2: defaultLogsLocalTableV2, LogsAttributeKeysTable: defaultLogAttributeKeysTable, LogsResourceKeysTable: defaultLogResourceKeysTable, LogsTagAttributeTable: defaultLogTagAttributeTable, @@ -168,6 +169,11 @@ func NewOptions( WriteBatchSize: defaultWriteBatchSize, Encoding: defaultEncoding, Connector: defaultConnector, + + LogsTableV2: defaultLogsTableV2, + LogsLocalTableV2: defaultLogsLocalTableV2, + LogsResourceTableV2: defaultLogsResourceTableV2, + LogsResourceLocalTableV2: defaultLogsResourceLocalTableV2, }, others: make(map[string]*namespaceConfig, len(otherNamespaces)), } diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 22f72a679f..911c1e0d40 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -117,9 +117,7 @@ type ClickHouseReader struct { topLevelOperationsTable string logsDB string logsTable string - logsTableV2 string logsLocalTable string - logsLocalTableV2 string logsAttributeKeys string logsResourceKeys string logsTagAttributeTable string @@ -128,6 +126,11 @@ type ClickHouseReader struct { fanoutStorage *storage.Storage queryProgressTracker queryprogress.QueryProgressTracker + logsTableV2 string + logsLocalTableV2 string + logsResourceTableV2 string + logsResourceLocalTableV2 string + promConfigFile string promConfig *config.Config alertManager am.Manager @@ -217,9 +220,7 @@ func NewReaderFromClickhouseConnection( topLevelOperationsTable: options.primary.TopLevelOperationsTable, logsDB: options.primary.LogsDB, logsTable: options.primary.LogsTable, - logsTableV2: options.primary.LogsTableV2, logsLocalTable: options.primary.LogsLocalTable, - logsLocalTableV2: options.primary.LogsLocalTableV2, logsAttributeKeys: options.primary.LogsAttributeKeysTable, logsResourceKeys: options.primary.LogsResourceKeysTable, logsTagAttributeTable: options.primary.LogsTagAttributeTable, @@ -228,7 +229,13 @@ func NewReaderFromClickhouseConnection( featureFlags: featureFlag, cluster: cluster, queryProgressTracker: queryprogress.NewQueryProgressTracker(), - UseLogsNewSchema: useLogsNewSchema, + + UseLogsNewSchema: useLogsNewSchema, + + logsTableV2: options.primary.LogsTableV2, + logsLocalTableV2: options.primary.LogsLocalTableV2, + logsResourceTableV2: options.primary.LogsResourceTableV2, + logsResourceLocalTableV2: options.primary.LogsResourceLocalTableV2, } } @@ -5373,7 +5380,7 @@ func (r *ClickHouseReader) LiveTailLogsV4(ctx context.Context, query string, tim bucketStart := (timestampStart / NANOSECOND) - 1800 // we have to form the query differently if the resource filters are used - if strings.Contains(query, defaultLogsResourceV2) { + if strings.Contains(query, r.logsResourceTableV2) { tmpQuery = fmt.Sprintf("seen_at_ts_bucket_start >=%d)) AND ts_bucket_start >=%d AND timestamp >=%d", bucketStart, bucketStart, timestampStart) } else { tmpQuery = fmt.Sprintf("ts_bucket_start >=%d AND timestamp >=%d", bucketStart, timestampStart) From 47bda3240f8e8569407084cab34bfccd4b896b95 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 12 Sep 2024 14:06:15 +0530 Subject: [PATCH 14/17] feat: cleanup and use flag --- pkg/query-service/app/querier/querier.go | 16 +++++++--------- pkg/query-service/app/querier/v2/querier.go | 16 +++++++--------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index e51246692b..c819012bfc 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -568,15 +568,13 @@ func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangePar func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) ([]*v3.Result, map[string]error, error) { // List query has support for only one query. - if params.CompositeQuery != nil { - if len(params.CompositeQuery.BuilderQueries) == 1 { - for _, v := range params.CompositeQuery.BuilderQueries { - // only allow of logs queries with timestamp ordering desc - if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { - startEndArr := getLogsListTsRanges(params.Start, params.End) - if len(startEndArr) > 0 { - return q.runLogsListQuery(ctx, params, keys, startEndArr) - } + if q.UseLogsNewSchema && params.CompositeQuery != nil { + for _, v := range params.CompositeQuery.BuilderQueries { + // only allow of logs queries with timestamp ordering desc + if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { + startEndArr := getLogsListTsRanges(params.Start, params.End) + if len(startEndArr) > 0 { + return q.runLogsListQuery(ctx, params, keys, startEndArr) } } } diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 829fcd1dc8..19b7da4efc 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -577,15 +577,13 @@ func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangePar func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) ([]*v3.Result, map[string]error, error) { // List query has support for only one query. - if params.CompositeQuery != nil { - if len(params.CompositeQuery.BuilderQueries) == 1 { - for _, v := range params.CompositeQuery.BuilderQueries { - // only allow of logs queries with timestamp ordering desc - if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { - startEndArr := getLogsListTsRanges(params.Start, params.End) - if len(startEndArr) > 0 { - return q.runLogsListQuery(ctx, params, keys, startEndArr) - } + if q.UseLogsNewSchema && params.CompositeQuery != nil { + for _, v := range params.CompositeQuery.BuilderQueries { + // only allow of logs queries with timestamp ordering desc + if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { + startEndArr := getLogsListTsRanges(params.Start, params.End) + if len(startEndArr) > 0 { + return q.runLogsListQuery(ctx, params, keys, startEndArr) } } } From 803d480596ddb916d88146fc09c9b8997a1d0d8b Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 12 Sep 2024 15:31:02 +0530 Subject: [PATCH 15/17] feat: restrict new list api to single query --- pkg/query-service/app/querier/querier.go | 2 +- pkg/query-service/app/querier/v2/querier.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index c819012bfc..a274699940 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -568,7 +568,7 @@ func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangePar func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) ([]*v3.Result, map[string]error, error) { // List query has support for only one query. - if q.UseLogsNewSchema && params.CompositeQuery != nil { + if q.UseLogsNewSchema && params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 { for _, v := range params.CompositeQuery.BuilderQueries { // only allow of logs queries with timestamp ordering desc if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 19b7da4efc..9b0691f2cd 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -577,7 +577,7 @@ func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangePar func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) ([]*v3.Result, map[string]error, error) { // List query has support for only one query. - if q.UseLogsNewSchema && params.CompositeQuery != nil { + if q.UseLogsNewSchema && params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 { for _, v := range params.CompositeQuery.BuilderQueries { // only allow of logs queries with timestamp ordering desc if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { From 9beb5290e6a0f05decede718cb44863d2b14e754 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 12 Sep 2024 20:39:42 +0530 Subject: [PATCH 16/17] fix: move getTsRanges to utils --- pkg/query-service/app/querier/querier.go | 41 +--------------- pkg/query-service/app/querier/querier_test.go | 45 ----------------- pkg/query-service/app/querier/v2/querier.go | 41 +--------------- .../app/querier/v2/querier_test.go | 46 ----------------- pkg/query-service/utils/logs.go | 38 ++++++++++++++ pkg/query-service/utils/logs_test.go | 49 +++++++++++++++++++ 6 files changed, 91 insertions(+), 169 deletions(-) create mode 100644 pkg/query-service/utils/logs.go create mode 100644 pkg/query-service/utils/logs_test.go diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index a274699940..2113b3f8fc 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -470,44 +470,7 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } -type logsListTsRange struct { - Start int64 - End int64 -} - -const HOUR_NANO = int64(3600000000000) - -func getLogsListTsRanges(start, end int64) []logsListTsRange { - startNano := utils.GetEpochNanoSecs(start) - endNano := utils.GetEpochNanoSecs(end) - result := []logsListTsRange{} - - if endNano-startNano > HOUR_NANO { - bucket := HOUR_NANO - tStartNano := endNano - bucket - - complete := false - for { - result = append(result, logsListTsRange{Start: tStartNano, End: endNano}) - if complete { - break - } - - bucket = bucket * 2 - endNano = tStartNano - tStartNano = tStartNano - bucket - - // break condition - if tStartNano <= startNano { - complete = true - tStartNano = startNano - } - } - } - return result -} - -func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []logsListTsRange) ([]*v3.Result, map[string]error, error) { +func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { res := make([]*v3.Result, 0) qName := "" pageSize := uint64(0) @@ -572,7 +535,7 @@ func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRan for _, v := range params.CompositeQuery.BuilderQueries { // only allow of logs queries with timestamp ordering desc if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { - startEndArr := getLogsListTsRanges(params.Start, params.End) + startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) if len(startEndArr) > 0 { return q.runLogsListQuery(ctx, params, keys, startEndArr) } diff --git a/pkg/query-service/app/querier/querier_test.go b/pkg/query-service/app/querier/querier_test.go index 0b6d5710ae..aecb7b27ba 100644 --- a/pkg/query-service/app/querier/querier_test.go +++ b/pkg/query-service/app/querier/querier_test.go @@ -1055,48 +1055,3 @@ func TestQueryRangeValueTypePromQL(t *testing.T) { } } } -func TestLogsListTsRange(t *testing.T) { - startEndData := []struct { - name string - start int64 - end int64 - res []logsListTsRange - }{ - { - name: "testing for less then one hour", - start: 1722262800000000000, // July 29, 2024 7:50:00 PM - end: 1722263800000000000, // July 29, 2024 8:06:40 PM - res: []logsListTsRange{}, - }, - { - name: "testing for more than one hour", - start: 1722255800000000000, // July 29, 2024 5:53:20 PM - end: 1722262800000000000, // July 29, 2024 8:06:40 PM - res: []logsListTsRange{ - {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM - {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM - }, - }, - { - name: "testing for 1 day", - start: 1722171576000000000, - end: 1722262800000000000, - res: []logsListTsRange{ - {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM - {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM - {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM - {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM - {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM - }, - }, - } - - for _, test := range startEndData { - res := getLogsListTsRanges(test.start, test.end) - for i, v := range res { - if test.res[i].Start != v.Start || test.res[i].End != v.End { - t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) - } - } - } -} diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 9b0691f2cd..b6d92faa44 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -479,44 +479,7 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } -type logsListTsRange struct { - Start int64 - End int64 -} - -const HOUR_NANO = int64(3600000000000) - -func getLogsListTsRanges(start, end int64) []logsListTsRange { - startNano := utils.GetEpochNanoSecs(start) - endNano := utils.GetEpochNanoSecs(end) - result := []logsListTsRange{} - - if endNano-startNano > HOUR_NANO { - bucket := HOUR_NANO - tStartNano := endNano - bucket - - complete := false - for { - result = append(result, logsListTsRange{Start: tStartNano, End: endNano}) - if complete { - break - } - - bucket = bucket * 2 - endNano = tStartNano - tStartNano = tStartNano - bucket - - // break condition - if tStartNano <= startNano { - complete = true - tStartNano = startNano - } - } - } - return result -} - -func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []logsListTsRange) ([]*v3.Result, map[string]error, error) { +func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { res := make([]*v3.Result, 0) qName := "" pageSize := uint64(0) @@ -581,7 +544,7 @@ func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRan for _, v := range params.CompositeQuery.BuilderQueries { // only allow of logs queries with timestamp ordering desc if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { - startEndArr := getLogsListTsRanges(params.Start, params.End) + startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) if len(startEndArr) > 0 { return q.runLogsListQuery(ctx, params, keys, startEndArr) } diff --git a/pkg/query-service/app/querier/v2/querier_test.go b/pkg/query-service/app/querier/v2/querier_test.go index 8ae1f28c58..5707e9f70d 100644 --- a/pkg/query-service/app/querier/v2/querier_test.go +++ b/pkg/query-service/app/querier/v2/querier_test.go @@ -1107,49 +1107,3 @@ func TestV2QueryRangeValueTypePromQL(t *testing.T) { } } } - -func TestLogsListTsRange(t *testing.T) { - startEndData := []struct { - name string - start int64 - end int64 - res []logsListTsRange - }{ - { - name: "testing for less then one hour", - start: 1722262800000000000, // July 29, 2024 7:50:00 PM - end: 1722263800000000000, // July 29, 2024 8:06:40 PM - res: []logsListTsRange{}, - }, - { - name: "testing for more than one hour", - start: 1722255800000000000, // July 29, 2024 5:53:20 PM - end: 1722262800000000000, // July 29, 2024 8:06:40 PM - res: []logsListTsRange{ - {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM - {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM - }, - }, - { - name: "testing for 1 day", - start: 1722171576000000000, - end: 1722262800000000000, - res: []logsListTsRange{ - {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM - {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM - {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM - {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM - {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM - }, - }, - } - - for _, test := range startEndData { - res := getLogsListTsRanges(test.start, test.end) - for i, v := range res { - if test.res[i].Start != v.Start || test.res[i].End != v.End { - t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) - } - } - } -} diff --git a/pkg/query-service/utils/logs.go b/pkg/query-service/utils/logs.go new file mode 100644 index 0000000000..2f536ef857 --- /dev/null +++ b/pkg/query-service/utils/logs.go @@ -0,0 +1,38 @@ +package utils + +const HOUR_NANO = int64(3600000000000) + +type LogsListTsRange struct { + Start int64 + End int64 +} + +func GetLogsListTsRanges(start, end int64) []LogsListTsRange { + startNano := GetEpochNanoSecs(start) + endNano := GetEpochNanoSecs(end) + result := []LogsListTsRange{} + + if endNano-startNano > HOUR_NANO { + bucket := HOUR_NANO + tStartNano := endNano - bucket + + complete := false + for { + result = append(result, LogsListTsRange{Start: tStartNano, End: endNano}) + if complete { + break + } + + bucket = bucket * 2 + endNano = tStartNano + tStartNano = tStartNano - bucket + + // break condition + if tStartNano <= startNano { + complete = true + tStartNano = startNano + } + } + } + return result +} diff --git a/pkg/query-service/utils/logs_test.go b/pkg/query-service/utils/logs_test.go new file mode 100644 index 0000000000..939fa5fa1b --- /dev/null +++ b/pkg/query-service/utils/logs_test.go @@ -0,0 +1,49 @@ +package utils + +import "testing" + +func TestLogsListTsRange(t *testing.T) { + startEndData := []struct { + name string + start int64 + end int64 + res []LogsListTsRange + }{ + { + name: "testing for less then one hour", + start: 1722262800000000000, // July 29, 2024 7:50:00 PM + end: 1722263800000000000, // July 29, 2024 8:06:40 PM + res: []LogsListTsRange{}, + }, + { + name: "testing for more than one hour", + start: 1722255800000000000, // July 29, 2024 5:53:20 PM + end: 1722262800000000000, // July 29, 2024 8:06:40 PM + res: []LogsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM + }, + }, + { + name: "testing for 1 day", + start: 1722171576000000000, + end: 1722262800000000000, + res: []LogsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM + {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM + {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM + {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM + }, + }, + } + + for _, test := range startEndData { + res := GetLogsListTsRanges(test.start, test.end) + for i, v := range res { + if test.res[i].Start != v.Start || test.res[i].End != v.End { + t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) + } + } + } +} From 077c7728b21c129e80e8029054d4af78383eb513 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 12 Sep 2024 23:28:43 +0530 Subject: [PATCH 17/17] fix: address pr comments --- .../app/clickhouseReader/reader.go | 252 +++++++++--------- pkg/query-service/app/http_handler.go | 139 +++++++--- .../app/logs/v4/query_builder.go | 5 +- 3 files changed, 229 insertions(+), 167 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index d30ec79cc2..a3d2536aa8 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -140,6 +140,7 @@ type ClickHouseReader struct { cluster string useLogsNewSchema bool + logsTableName string } // NewTraceReader returns a TraceReader for the database @@ -203,6 +204,11 @@ func NewReaderFromClickhouseConnection( }, } + logsTableName := options.primary.LogsTable + if useLogsNewSchema { + logsTableName = options.primary.LogsTableV2 + } + return &ClickHouseReader{ db: wrap, localDB: localDB, @@ -236,6 +242,7 @@ func NewReaderFromClickhouseConnection( logsLocalTableV2: options.primary.LogsLocalTableV2, logsResourceTableV2: options.primary.LogsResourceTableV2, logsResourceLocalTableV2: options.primary.LogsResourceLocalTableV2, + logsTableName: logsTableName, } } @@ -3530,11 +3537,7 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe resources = removeUnderscoreDuplicateFields(resources) statements := []model.ShowCreateTableStatement{} - table := r.logsLocalTable - if r.useLogsNewSchema { - table = r.logsLocalTableV2 - } - query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, table) + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsTable) err = r.db.Select(ctx, &statements, query) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} @@ -3565,6 +3568,72 @@ func isSelectedField(tableStatement string, field model.LogField) bool { return strings.Contains(tableStatement, name) } +func (r *ClickHouseReader) UpdateLogFieldV2(ctx context.Context, field *model.UpdateField) *model.ApiError { + if !field.Selected { + return model.ForbiddenError(errors.New("removing a selected field is not allowed, please reach out to support.")) + } + + colname := utils.GetClickhouseColumnNameV2(field.Type, field.DataType, field.Name) + + dataType := strings.ToLower(field.DataType) + if dataType == "int64" || dataType == "float64" { + dataType = "number" + } + attrColName := fmt.Sprintf("%s_%s", field.Type, dataType) + for _, table := range []string{r.logsLocalTableV2, r.logsTableV2} { + q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS `%s` %s DEFAULT %s['%s'] CODEC(ZSTD(1))" + query := fmt.Sprintf(q, + r.logsDB, table, + r.cluster, + colname, field.DataType, + attrColName, + field.Name, + ) + err := r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + + query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS `%s_exists` bool DEFAULT if(mapContains(%s, '%s') != 0, true, false) CODEC(ZSTD(1))", + r.logsDB, table, + r.cluster, + colname, + attrColName, + field.Name, + ) + err = r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + } + + // create the index + if strings.ToLower(field.DataType) == "bool" { + // there is no point in creating index for bool attributes as the cardinality is just 2 + return nil + } + + if field.IndexType == "" { + field.IndexType = constants.DefaultLogSkipIndexType + } + if field.IndexGranularity == 0 { + field.IndexGranularity = constants.DefaultLogSkipIndexGranularity + } + query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS `%s_idx` (`%s`) TYPE %s GRANULARITY %d", + r.logsDB, r.logsLocalTableV2, + r.cluster, + colname, + colname, + field.IndexType, + field.IndexGranularity, + ) + err := r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + return nil +} + func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError { // don't allow updating static fields if field.Type == constants.Static { @@ -3572,132 +3641,72 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda return &model.ApiError{Err: err, Typ: model.ErrorBadData} } + if r.useLogsNewSchema { + return r.UpdateLogFieldV2(ctx, field) + } + // if a field is selected it means that the field needs to be indexed if field.Selected { - // create materialized column - if r.useLogsNewSchema { - colname := utils.GetClickhouseColumnNameV2(field.Type, field.DataType, field.Name) + colname := utils.GetClickhouseColumnName(field.Type, field.DataType, field.Name) - dataType := strings.ToLower(field.DataType) - if dataType == "int64" || dataType == "float64" { - dataType = "number" - } - attrColName := fmt.Sprintf("%s_%s", field.Type, dataType) - for _, table := range []string{r.logsLocalTableV2, r.logsTableV2} { - q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS `%s` %s DEFAULT %s['%s'] CODEC(ZSTD(1))" - query := fmt.Sprintf(q, - r.logsDB, table, - r.cluster, - colname, field.DataType, - attrColName, - field.Name, - ) - err := r.db.Exec(ctx, query) - if err != nil { - return &model.ApiError{Err: err, Typ: model.ErrorInternal} - } + keyColName := fmt.Sprintf("%s_%s_key", field.Type, strings.ToLower(field.DataType)) + valueColName := fmt.Sprintf("%s_%s_value", field.Type, strings.ToLower(field.DataType)) - query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS `%s_exists` bool DEFAULT if(mapContains(%s, '%s') != 0, true, false) CODEC(ZSTD(1))", - r.logsDB, table, - r.cluster, - colname, - attrColName, - field.Name, - ) - err = r.db.Exec(ctx, query) - if err != nil { - return &model.ApiError{Err: err, Typ: model.ErrorInternal} - } - } - - // create the index - if strings.ToLower(field.DataType) == "bool" { - // there is no point in creating index for bool attributes as the cardinality is just 2 - return nil - } + // create materialized column - if field.IndexType == "" { - field.IndexType = constants.DefaultLogSkipIndexType - } - if field.IndexGranularity == 0 { - field.IndexGranularity = constants.DefaultLogSkipIndexGranularity - } - query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS `%s_idx` (`%s`) TYPE %s GRANULARITY %d", - r.logsDB, r.logsLocalTableV2, + for _, table := range []string{r.logsLocalTable, r.logsTable} { + q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s DEFAULT %s[indexOf(%s, '%s')] CODEC(ZSTD(1))" + query := fmt.Sprintf(q, + r.logsDB, table, r.cluster, - colname, - colname, - field.IndexType, - field.IndexGranularity, + colname, field.DataType, + valueColName, + keyColName, + field.Name, ) err := r.db.Exec(ctx, query) if err != nil { return &model.ApiError{Err: err, Typ: model.ErrorInternal} } - } else { - // old schema mat columns - - colname := utils.GetClickhouseColumnName(field.Type, field.DataType, field.Name) - - keyColName := fmt.Sprintf("%s_%s_key", field.Type, strings.ToLower(field.DataType)) - valueColName := fmt.Sprintf("%s_%s_value", field.Type, strings.ToLower(field.DataType)) - - // create materialized column - - for _, table := range []string{r.logsLocalTable, r.logsTable} { - q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s DEFAULT %s[indexOf(%s, '%s')] CODEC(ZSTD(1))" - query := fmt.Sprintf(q, - r.logsDB, table, - r.cluster, - colname, field.DataType, - valueColName, - keyColName, - field.Name, - ) - err := r.db.Exec(ctx, query) - if err != nil { - return &model.ApiError{Err: err, Typ: model.ErrorInternal} - } - - query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists` bool DEFAULT if(indexOf(%s, '%s') != 0, true, false) CODEC(ZSTD(1))", - r.logsDB, table, - r.cluster, - strings.TrimSuffix(colname, "`"), - keyColName, - field.Name, - ) - err = r.db.Exec(ctx, query) - if err != nil { - return &model.ApiError{Err: err, Typ: model.ErrorInternal} - } - } - - // create the index - if strings.ToLower(field.DataType) == "bool" { - // there is no point in creating index for bool attributes as the cardinality is just 2 - return nil - } - if field.IndexType == "" { - field.IndexType = constants.DefaultLogSkipIndexType - } - if field.IndexGranularity == 0 { - field.IndexGranularity = constants.DefaultLogSkipIndexGranularity - } - query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS %s_idx` (%s) TYPE %s GRANULARITY %d", - r.logsDB, r.logsLocalTable, + query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists` bool DEFAULT if(indexOf(%s, '%s') != 0, true, false) CODEC(ZSTD(1))", + r.logsDB, table, r.cluster, strings.TrimSuffix(colname, "`"), - colname, - field.IndexType, - field.IndexGranularity, + keyColName, + field.Name, ) - err := r.db.Exec(ctx, query) + err = r.db.Exec(ctx, query) if err != nil { return &model.ApiError{Err: err, Typ: model.ErrorInternal} } } + // create the index + if strings.ToLower(field.DataType) == "bool" { + // there is no point in creating index for bool attributes as the cardinality is just 2 + return nil + } + + if field.IndexType == "" { + field.IndexType = constants.DefaultLogSkipIndexType + } + if field.IndexGranularity == 0 { + field.IndexGranularity = constants.DefaultLogSkipIndexGranularity + } + query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS %s_idx` (%s) TYPE %s GRANULARITY %d", + r.logsDB, r.logsLocalTable, + r.cluster, + strings.TrimSuffix(colname, "`"), + colname, + field.IndexType, + field.IndexGranularity, + ) + err := r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + } else { // We are not allowing to delete a materialized column // For more details please check https://github.com/SigNoz/signoz/issues/4566 @@ -4293,11 +4302,7 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v defer rows.Close() statements := []model.ShowCreateTableStatement{} - table := r.logsLocalTable - if r.useLogsNewSchema { - table = r.logsLocalTableV2 - } - query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, table) + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsTable) err = r.db.Select(ctx, &statements, query) if err != nil { return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) @@ -4351,11 +4356,7 @@ func (r *ClickHouseReader) GetLogAttributeKeys(ctx context.Context, req *v3.Filt defer rows.Close() statements := []model.ShowCreateTableStatement{} - table := r.logsLocalTable - if r.useLogsNewSchema { - table = r.logsLocalTableV2 - } - query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, table) + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsTable) err = r.db.Select(ctx, &statements, query) if err != nil { return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) @@ -4429,10 +4430,6 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi } searchText := fmt.Sprintf("%%%s%%", req.SearchText) - table := r.logsLocalTable - if r.useLogsNewSchema { - table = r.logsLocalTableV2 - } // check if the tagKey is a topLevelColumn if _, ok := constants.StaticFieldsLogsV3[req.FilterAttributeKey]; ok { @@ -4446,10 +4443,10 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi // prepare the query and run if len(req.SearchText) != 0 { - query = fmt.Sprintf("select distinct %s from %s.%s where timestamp >= toInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)*1000000000) and %s ILIKE $1 limit $2", selectKey, r.logsDB, table, filterValueColumnWhere) + query = fmt.Sprintf("select distinct %s from %s.%s where timestamp >= toInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)*1000000000) and %s ILIKE $1 limit $2", selectKey, r.logsDB, r.logsTable, filterValueColumnWhere) rows, err = r.db.Query(ctx, query, searchText, req.Limit) } else { - query = fmt.Sprintf("select distinct %s from %s.%s where timestamp >= toInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)*1000000000) limit $1", selectKey, r.logsDB, table) + query = fmt.Sprintf("select distinct %s from %s.%s where timestamp >= toInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)*1000000000) limit $1", selectKey, r.logsDB, r.logsTable) rows, err = r.db.Query(ctx, query, req.Limit) } } else if len(req.SearchText) != 0 { @@ -5388,6 +5385,7 @@ func (r *ClickHouseReader) LiveTailLogsV4(ctx context.Context, query string, tim if idStart != "" { tmpQuery = fmt.Sprintf("%s AND id > '%s'", tmpQuery, idStart) } + // the reason we are doing desc is that we need the latest logs first tmpQuery = query + tmpQuery + " order by timestamp desc, id desc limit 100" diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 20a1b90455..8f4cf25233 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -3932,7 +3932,7 @@ func (aH *APIHandler) GetQueryProgressUpdates(w http.ResponseWriter, r *http.Req } } -func (aH *APIHandler) liveTailLogs(w http.ResponseWriter, r *http.Request) { +func (aH *APIHandler) liveTailLogsV2(w http.ResponseWriter, r *http.Request) { // get the param from url and add it to body stringReader := strings.NewReader(r.URL.Query().Get("q")) @@ -3991,48 +3991,109 @@ func (aH *APIHandler) liveTailLogs(w http.ResponseWriter, r *http.Request) { flusher.Flush() // create the client + client := &v3.LogsLiveTailClientV2{Name: r.RemoteAddr, Logs: make(chan *model.SignozLogV2, 1000), Done: make(chan *bool), Error: make(chan error)} + go aH.reader.LiveTailLogsV4(r.Context(), queryString, uint64(queryRangeParams.Start), "", client) + for { + select { + case log := <-client.Logs: + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.Encode(log) + fmt.Fprintf(w, "data: %v\n\n", buf.String()) + flusher.Flush() + case <-client.Done: + zap.L().Debug("done!") + return + case err := <-client.Error: + zap.L().Error("error occurred", zap.Error(err)) + fmt.Fprintf(w, "event: error\ndata: %v\n\n", err.Error()) + flusher.Flush() + return + } + } + +} + +func (aH *APIHandler) liveTailLogs(w http.ResponseWriter, r *http.Request) { if aH.UseLogsNewSchema { - // create the client - client := &v3.LogsLiveTailClientV2{Name: r.RemoteAddr, Logs: make(chan *model.SignozLogV2, 1000), Done: make(chan *bool), Error: make(chan error)} - go aH.reader.LiveTailLogsV4(r.Context(), queryString, uint64(queryRangeParams.Start), "", client) - for { - select { - case log := <-client.Logs: - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.Encode(log) - fmt.Fprintf(w, "data: %v\n\n", buf.String()) - flusher.Flush() - case <-client.Done: - zap.L().Debug("done!") - return - case err := <-client.Error: - zap.L().Error("error occurred", zap.Error(err)) - fmt.Fprintf(w, "event: error\ndata: %v\n\n", err.Error()) - flusher.Flush() + aH.liveTailLogsV2(w, r) + return + } + + // get the param from url and add it to body + stringReader := strings.NewReader(r.URL.Query().Get("q")) + r.Body = io.NopCloser(stringReader) + + queryRangeParams, apiErrorObj := ParseQueryRangeParams(r) + if apiErrorObj != nil { + zap.L().Error(apiErrorObj.Err.Error()) + RespondError(w, apiErrorObj, nil) + return + } + + var err error + var queryString string + switch queryRangeParams.CompositeQuery.QueryType { + case v3.QueryTypeBuilder: + // check if any enrichment is required for logs if yes then enrich them + if logsv3.EnrichmentRequired(queryRangeParams) { + // get the fields if any logs query is present + var fields map[string]v3.AttributeKey + fields, err = aH.getLogFieldsV3(r.Context(), queryRangeParams) + if err != nil { + apiErrObj := &model.ApiError{Typ: model.ErrorInternal, Err: err} + RespondError(w, apiErrObj, nil) return } + logsv3.Enrich(queryRangeParams, fields) } - } else { - client := &v3.LogsLiveTailClient{Name: r.RemoteAddr, Logs: make(chan *model.SignozLog, 1000), Done: make(chan *bool), Error: make(chan error)} - go aH.reader.LiveTailLogsV3(r.Context(), queryString, uint64(queryRangeParams.Start), "", client) - for { - select { - case log := <-client.Logs: - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.Encode(log) - fmt.Fprintf(w, "data: %v\n\n", buf.String()) - flusher.Flush() - case <-client.Done: - zap.L().Debug("done!") - return - case err := <-client.Error: - zap.L().Error("error occurred", zap.Error(err)) - fmt.Fprintf(w, "event: error\ndata: %v\n\n", err.Error()) - flusher.Flush() - return - } + + queryString, err = aH.queryBuilder.PrepareLiveTailQuery(queryRangeParams) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + + default: + err = fmt.Errorf("invalid query type") + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + + // create the client + client := &v3.LogsLiveTailClient{Name: r.RemoteAddr, Logs: make(chan *model.SignozLog, 1000), Done: make(chan *bool), Error: make(chan error)} + go aH.reader.LiveTailLogsV3(r.Context(), queryString, uint64(queryRangeParams.Start), "", client) + + w.Header().Set("Connection", "keep-alive") + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Access-Control-Allow-Origin", "*") + w.WriteHeader(200) + + flusher, ok := w.(http.Flusher) + if !ok { + err := model.ApiError{Typ: model.ErrorStreamingNotSupported, Err: nil} + RespondError(w, &err, "streaming is not supported") + return + } + // flush the headers + flusher.Flush() + for { + select { + case log := <-client.Logs: + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.Encode(log) + fmt.Fprintf(w, "data: %v\n\n", buf.String()) + flusher.Flush() + case <-client.Done: + zap.L().Debug("done!") + return + case err := <-client.Error: + zap.L().Error("error occurred", zap.Error(err)) + fmt.Fprintf(w, "event: error\ndata: %v\n\n", err.Error()) + flusher.Flush() + return } } diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index b96c5b9113..4b9b2446f9 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -451,7 +451,10 @@ func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) { } // join both the filter clauses if resourceSubQuery != "" { - filterSubQuery = filterSubQuery + " AND (resource_fingerprint GLOBAL IN " + resourceSubQuery + if filterSubQuery != "" { + filterSubQuery = filterSubQuery + " AND (resource_fingerprint GLOBAL IN " + resourceSubQuery + } + filterSubQuery = "(resource_fingerprint GLOBAL IN " + resourceSubQuery } // the reader will add the timestamp and id filters