Skip to content

Commit

Permalink
fix: exists check for json filters added (#3675)
Browse files Browse the repository at this point in the history 
* fix: exists check for json filters added

* fix: comment updated
  • Loading branch information
@nityanandagohain
nityanandagohain authored Oct 6, 2023
1 parent a306fb6 commit abed60b
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 19 deletions.
42 changes: 32 additions & 10 deletions pkg/query-service/app/logs/v3/json_filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,13 @@ var jsonLogOperators = map[v3.FilterOperator]string{
v3.FilterOperatorNotRegex: "NOT match(%s, %s)",
v3.FilterOperatorIn: "IN",
v3.FilterOperatorNotIn: "NOT IN",
v3.FilterOperatorExists: "JSON_EXISTS(%s, '$.%s')",
v3.FilterOperatorNotExists: "NOT JSON_EXISTS(%s, '$.%s')",
v3.FilterOperatorHas: "has(%s, %s)",
v3.FilterOperatorNotHas: "NOT has(%s, %s)",
}

func getJSONFilterKey(key v3.AttributeKey, isArray bool) (string, error) {
func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) {
keyArr := strings.Split(key.Key, ".")
if len(keyArr) < 2 {
return "", fmt.Errorf("incorrect key, should contain at least 2 parts")
Expand All @@ -67,6 +69,10 @@ func getJSONFilterKey(key v3.AttributeKey, isArray bool) (string, error) {
return "", fmt.Errorf("only body can be the root key")
}

if op == v3.FilterOperatorExists || op == v3.FilterOperatorNotExists {
return keyArr[0], nil
}

var dataType string
var ok bool
if dataType, ok = dataTypeMapping[string(key.DataType)]; !ok {
Expand Down Expand Up @@ -99,29 +105,45 @@ func GetJSONFilter(item v3.FilterItem) (string, error) {
dataType = v3.AttributeKeyDataType(val)
}

key, err := getJSONFilterKey(item.Key, isArray)
key, err := getJSONFilterKey(item.Key, item.Operator, isArray)
if err != nil {
return "", err
}

// non array
value, err := utils.ValidateAndCastValue(item.Value, dataType)
if err != nil {
return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err)
op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator))))

var value interface{}
if op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists {
value, err = utils.ValidateAndCastValue(item.Value, dataType)
if err != nil {
return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err)
}
}

op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator))))
var filter string
if logsOp, ok := jsonLogOperators[op]; ok {
switch op {
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
filter = fmt.Sprintf(logsOp, key, strings.Join(strings.Split(item.Key.Key, ".")[1:], "."))
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas:
fmtVal := utils.ClickHouseFormattedValue(value)
return fmt.Sprintf(logsOp, key, fmtVal), nil
filter = fmt.Sprintf(logsOp, key, fmtVal)
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
return fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value), nil
filter = fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value)
default:
fmtVal := utils.ClickHouseFormattedValue(value)
return fmt.Sprintf("%s %s %s", key, logsOp, fmtVal), nil
filter = fmt.Sprintf("%s %s %s", key, logsOp, fmtVal)
}
} else {
return "", fmt.Errorf("unsupported operator: %s", op)
}
return "", fmt.Errorf("unsupported operator: %s", op)

// add exists check for non array items as default values of int/float/bool will corrupt the results
if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) {
existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", strings.Join(strings.Split(item.Key.Key, ".")[1:], "."))
filter = fmt.Sprintf("%s AND %s", existsFilter, filter)
}

return filter, nil
}
30 changes: 22 additions & 8 deletions pkg/query-service/app/logs/v3/json_filter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ var testGetJSONFilterKeyData = []struct {
Key v3.AttributeKey
IsArray bool
ClickhouseKey string
Operator v3.FilterOperator
Error bool
}{
{
Expand Down Expand Up @@ -129,7 +130,7 @@ var testGetJSONFilterKeyData = []struct {
func TestGetJSONFilterKey(t *testing.T) {
for _, tt := range testGetJSONFilterKeyData {
Convey("testgetKey", t, func() {
columnName, err := getJSONFilterKey(tt.Key, tt.IsArray)
columnName, err := getJSONFilterKey(tt.Key, tt.Operator, tt.IsArray)
if tt.Error {
So(err, ShouldNotBeNil)
} else {
Expand Down Expand Up @@ -209,7 +210,7 @@ var testGetJSONFilterData = []struct {
Operator: "=",
Value: "hello",
},
Filter: "JSON_VALUE(body, '$.message') = 'hello'",
Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') = 'hello'",
},
{
Name: "eq operator number",
Expand All @@ -222,7 +223,7 @@ var testGetJSONFilterData = []struct {
Operator: "=",
Value: 1,
},
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1",
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1",
},
{
Name: "neq operator number",
Expand All @@ -235,7 +236,7 @@ var testGetJSONFilterData = []struct {
Operator: "=",
Value: 1.1,
},
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000",
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000",
},
{
Name: "eq operator bool",
Expand All @@ -248,7 +249,7 @@ var testGetJSONFilterData = []struct {
Operator: "=",
Value: true,
},
Filter: "JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true",
Filter: "JSON_EXISTS(body, '$.boolkey') AND JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true",
},
{
Name: "greater than operator",
Expand All @@ -261,7 +262,7 @@ var testGetJSONFilterData = []struct {
Operator: ">",
Value: 1,
},
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1",
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1",
},
{
Name: "regex operator",
Expand All @@ -274,7 +275,7 @@ var testGetJSONFilterData = []struct {
Operator: "regex",
Value: "a*",
},
Filter: "match(JSON_VALUE(body, '$.message'), 'a*')",
Filter: "JSON_EXISTS(body, '$.message') AND match(JSON_VALUE(body, '$.message'), 'a*')",
},
{
Name: "contains operator",
Expand All @@ -287,7 +288,20 @@ var testGetJSONFilterData = []struct {
Operator: "contains",
Value: "a",
},
Filter: "JSON_VALUE(body, '$.message') ILIKE '%a%'",
Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%'",
},
{
Name: "exists",
FilterItem: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.message",
DataType: "string",
IsJSON: true,
},
Operator: "exists",
Value: "",
},
Filter: "JSON_EXISTS(body, '$.message')",
},
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/query-service/app/logs/v3/query_builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -908,7 +908,7 @@ var testBuildLogsQueryData = []struct {
},
},
TableName: "logs",
ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC",
ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC",
},
{
Name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy",
Expand Down

0 comments on commit abed60b

Please sign in to comment.