From 90b5f884135121bd2fcb3f6775269a959c8820e3 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Thu, 12 Sep 2024 21:34:27 +0530 Subject: [PATCH] feat: logs list API, logic update for better perf (#5912) * feat: logsV4 initial refactoring * feat: filter_query builder with tests added * feat: all functions of v4 refactored * fix: tests fixed * feat: logs list API, logic update for better perf * fix: update select for table panel * fix: tests updated with better examples of limit and group by * fix: resource filter support in live tail * feat: cleanup and use flag * feat: restrict new list api to single query * fix: move getTsRanges to utils --- pkg/query-service/app/querier/querier.go | 72 +++++++++++++++++++++ pkg/query-service/app/querier/v2/querier.go | 72 +++++++++++++++++++++ pkg/query-service/utils/logs.go | 38 +++++++++++ pkg/query-service/utils/logs_test.go | 49 ++++++++++++++ 4 files changed, 231 insertions(+) create mode 100644 pkg/query-service/utils/logs.go create mode 100644 pkg/query-service/utils/logs_test.go diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 0663afd126..2113b3f8fc 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -15,6 +15,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" chErrors "go.signoz.io/signoz/pkg/query-service/errors" + "go.signoz.io/signoz/pkg/query-service/utils" "go.signoz.io/signoz/pkg/query-service/cache" "go.signoz.io/signoz/pkg/query-service/interfaces" @@ -469,7 +470,78 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } +func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { + res := make([]*v3.Result, 0) + qName := "" + pageSize := uint64(0) + + // se we are considering only one query + for name, v := range params.CompositeQuery.BuilderQueries { + qName = name + pageSize = v.PageSize + } + data := []*v3.Row{} + + for _, v := range tsRanges { + params.Start = v.Start + params.End = v.End + + params.CompositeQuery.BuilderQueries[qName].PageSize = pageSize - uint64(len(data)) + queries, err := q.builder.PrepareQueries(params, keys) + if err != nil { + return nil, nil, err + } + + // this will to run only once + for name, query := range queries { + rowList, err := q.reader.GetListResultV3(ctx, query) + if err != nil { + errs := []error{err} + errQuriesByName := map[string]error{ + name: err, + } + return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) + } + data = append(data, rowList...) + } + + // append a filter to the params + if len(data) > 0 { + params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "id", + IsColumn: true, + DataType: "string", + }, + Operator: v3.FilterOperatorLessThan, + Value: data[len(data)-1].Data["id"], + }) + } + + if uint64(len(data)) >= pageSize { + break + } + } + res = append(res, &v3.Result{ + QueryName: qName, + List: data, + }) + return res, nil, nil +} + func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) ([]*v3.Result, map[string]error, error) { + // List query has support for only one query. + if q.UseLogsNewSchema && params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 { + for _, v := range params.CompositeQuery.BuilderQueries { + // only allow of logs queries with timestamp ordering desc + if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { + startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) + if len(startEndArr) > 0 { + return q.runLogsListQuery(ctx, params, keys, startEndArr) + } + } + } + } queries, err := q.builder.PrepareQueries(params, keys) diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 01cbf6d649..b6d92faa44 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -15,6 +15,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" chErrors "go.signoz.io/signoz/pkg/query-service/errors" + "go.signoz.io/signoz/pkg/query-service/utils" "go.signoz.io/signoz/pkg/query-service/cache" "go.signoz.io/signoz/pkg/query-service/interfaces" @@ -478,7 +479,78 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } +func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { + res := make([]*v3.Result, 0) + qName := "" + pageSize := uint64(0) + + // se we are considering only one query + for name, v := range params.CompositeQuery.BuilderQueries { + qName = name + pageSize = v.PageSize + } + data := []*v3.Row{} + + for _, v := range tsRanges { + params.Start = v.Start + params.End = v.End + + params.CompositeQuery.BuilderQueries[qName].PageSize = pageSize - uint64(len(data)) + queries, err := q.builder.PrepareQueries(params, keys) + if err != nil { + return nil, nil, err + } + + // this will to run only once + for name, query := range queries { + rowList, err := q.reader.GetListResultV3(ctx, query) + if err != nil { + errs := []error{err} + errQuriesByName := map[string]error{ + name: err, + } + return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) + } + data = append(data, rowList...) + } + + // append a filter to the params + if len(data) > 0 { + params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "id", + IsColumn: true, + DataType: "string", + }, + Operator: v3.FilterOperatorLessThan, + Value: data[len(data)-1].Data["id"], + }) + } + + if uint64(len(data)) >= pageSize { + break + } + } + res = append(res, &v3.Result{ + QueryName: qName, + List: data, + }) + return res, nil, nil +} + func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) ([]*v3.Result, map[string]error, error) { + // List query has support for only one query. + if q.UseLogsNewSchema && params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 { + for _, v := range params.CompositeQuery.BuilderQueries { + // only allow of logs queries with timestamp ordering desc + if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { + startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) + if len(startEndArr) > 0 { + return q.runLogsListQuery(ctx, params, keys, startEndArr) + } + } + } + } queries, err := q.builder.PrepareQueries(params, keys) diff --git a/pkg/query-service/utils/logs.go b/pkg/query-service/utils/logs.go new file mode 100644 index 0000000000..2f536ef857 --- /dev/null +++ b/pkg/query-service/utils/logs.go @@ -0,0 +1,38 @@ +package utils + +const HOUR_NANO = int64(3600000000000) + +type LogsListTsRange struct { + Start int64 + End int64 +} + +func GetLogsListTsRanges(start, end int64) []LogsListTsRange { + startNano := GetEpochNanoSecs(start) + endNano := GetEpochNanoSecs(end) + result := []LogsListTsRange{} + + if endNano-startNano > HOUR_NANO { + bucket := HOUR_NANO + tStartNano := endNano - bucket + + complete := false + for { + result = append(result, LogsListTsRange{Start: tStartNano, End: endNano}) + if complete { + break + } + + bucket = bucket * 2 + endNano = tStartNano + tStartNano = tStartNano - bucket + + // break condition + if tStartNano <= startNano { + complete = true + tStartNano = startNano + } + } + } + return result +} diff --git a/pkg/query-service/utils/logs_test.go b/pkg/query-service/utils/logs_test.go new file mode 100644 index 0000000000..939fa5fa1b --- /dev/null +++ b/pkg/query-service/utils/logs_test.go @@ -0,0 +1,49 @@ +package utils + +import "testing" + +func TestLogsListTsRange(t *testing.T) { + startEndData := []struct { + name string + start int64 + end int64 + res []LogsListTsRange + }{ + { + name: "testing for less then one hour", + start: 1722262800000000000, // July 29, 2024 7:50:00 PM + end: 1722263800000000000, // July 29, 2024 8:06:40 PM + res: []LogsListTsRange{}, + }, + { + name: "testing for more than one hour", + start: 1722255800000000000, // July 29, 2024 5:53:20 PM + end: 1722262800000000000, // July 29, 2024 8:06:40 PM + res: []LogsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM + }, + }, + { + name: "testing for 1 day", + start: 1722171576000000000, + end: 1722262800000000000, + res: []LogsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM + {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM + {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM + {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM + }, + }, + } + + for _, test := range startEndData { + res := GetLogsListTsRanges(test.start, test.end) + for i, v := range res { + if test.res[i].Start != v.Start || test.res[i].End != v.End { + t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) + } + } + } +}