diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index a274699940..2113b3f8fc 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -470,44 +470,7 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } -type logsListTsRange struct { - Start int64 - End int64 -} - -const HOUR_NANO = int64(3600000000000) - -func getLogsListTsRanges(start, end int64) []logsListTsRange { - startNano := utils.GetEpochNanoSecs(start) - endNano := utils.GetEpochNanoSecs(end) - result := []logsListTsRange{} - - if endNano-startNano > HOUR_NANO { - bucket := HOUR_NANO - tStartNano := endNano - bucket - - complete := false - for { - result = append(result, logsListTsRange{Start: tStartNano, End: endNano}) - if complete { - break - } - - bucket = bucket * 2 - endNano = tStartNano - tStartNano = tStartNano - bucket - - // break condition - if tStartNano <= startNano { - complete = true - tStartNano = startNano - } - } - } - return result -} - -func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []logsListTsRange) ([]*v3.Result, map[string]error, error) { +func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { res := make([]*v3.Result, 0) qName := "" pageSize := uint64(0) @@ -572,7 +535,7 @@ func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRan for _, v := range params.CompositeQuery.BuilderQueries { // only allow of logs queries with timestamp ordering desc if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { - startEndArr := getLogsListTsRanges(params.Start, params.End) + startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) if len(startEndArr) > 0 { return q.runLogsListQuery(ctx, params, keys, startEndArr) } diff --git a/pkg/query-service/app/querier/querier_test.go b/pkg/query-service/app/querier/querier_test.go index 0b6d5710ae..aecb7b27ba 100644 --- a/pkg/query-service/app/querier/querier_test.go +++ b/pkg/query-service/app/querier/querier_test.go @@ -1055,48 +1055,3 @@ func TestQueryRangeValueTypePromQL(t *testing.T) { } } } -func TestLogsListTsRange(t *testing.T) { - startEndData := []struct { - name string - start int64 - end int64 - res []logsListTsRange - }{ - { - name: "testing for less then one hour", - start: 1722262800000000000, // July 29, 2024 7:50:00 PM - end: 1722263800000000000, // July 29, 2024 8:06:40 PM - res: []logsListTsRange{}, - }, - { - name: "testing for more than one hour", - start: 1722255800000000000, // July 29, 2024 5:53:20 PM - end: 1722262800000000000, // July 29, 2024 8:06:40 PM - res: []logsListTsRange{ - {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM - {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM - }, - }, - { - name: "testing for 1 day", - start: 1722171576000000000, - end: 1722262800000000000, - res: []logsListTsRange{ - {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM - {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM - {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM - {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM - {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM - }, - }, - } - - for _, test := range startEndData { - res := getLogsListTsRanges(test.start, test.end) - for i, v := range res { - if test.res[i].Start != v.Start || test.res[i].End != v.End { - t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) - } - } - } -} diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 9b0691f2cd..b6d92faa44 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -479,44 +479,7 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } -type logsListTsRange struct { - Start int64 - End int64 -} - -const HOUR_NANO = int64(3600000000000) - -func getLogsListTsRanges(start, end int64) []logsListTsRange { - startNano := utils.GetEpochNanoSecs(start) - endNano := utils.GetEpochNanoSecs(end) - result := []logsListTsRange{} - - if endNano-startNano > HOUR_NANO { - bucket := HOUR_NANO - tStartNano := endNano - bucket - - complete := false - for { - result = append(result, logsListTsRange{Start: tStartNano, End: endNano}) - if complete { - break - } - - bucket = bucket * 2 - endNano = tStartNano - tStartNano = tStartNano - bucket - - // break condition - if tStartNano <= startNano { - complete = true - tStartNano = startNano - } - } - } - return result -} - -func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []logsListTsRange) ([]*v3.Result, map[string]error, error) { +func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { res := make([]*v3.Result, 0) qName := "" pageSize := uint64(0) @@ -581,7 +544,7 @@ func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRan for _, v := range params.CompositeQuery.BuilderQueries { // only allow of logs queries with timestamp ordering desc if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { - startEndArr := getLogsListTsRanges(params.Start, params.End) + startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) if len(startEndArr) > 0 { return q.runLogsListQuery(ctx, params, keys, startEndArr) } diff --git a/pkg/query-service/app/querier/v2/querier_test.go b/pkg/query-service/app/querier/v2/querier_test.go index 8ae1f28c58..5707e9f70d 100644 --- a/pkg/query-service/app/querier/v2/querier_test.go +++ b/pkg/query-service/app/querier/v2/querier_test.go @@ -1107,49 +1107,3 @@ func TestV2QueryRangeValueTypePromQL(t *testing.T) { } } } - -func TestLogsListTsRange(t *testing.T) { - startEndData := []struct { - name string - start int64 - end int64 - res []logsListTsRange - }{ - { - name: "testing for less then one hour", - start: 1722262800000000000, // July 29, 2024 7:50:00 PM - end: 1722263800000000000, // July 29, 2024 8:06:40 PM - res: []logsListTsRange{}, - }, - { - name: "testing for more than one hour", - start: 1722255800000000000, // July 29, 2024 5:53:20 PM - end: 1722262800000000000, // July 29, 2024 8:06:40 PM - res: []logsListTsRange{ - {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM - {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM - }, - }, - { - name: "testing for 1 day", - start: 1722171576000000000, - end: 1722262800000000000, - res: []logsListTsRange{ - {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM - {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM - {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM - {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM - {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM - }, - }, - } - - for _, test := range startEndData { - res := getLogsListTsRanges(test.start, test.end) - for i, v := range res { - if test.res[i].Start != v.Start || test.res[i].End != v.End { - t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) - } - } - } -} diff --git a/pkg/query-service/utils/logs.go b/pkg/query-service/utils/logs.go new file mode 100644 index 0000000000..2f536ef857 --- /dev/null +++ b/pkg/query-service/utils/logs.go @@ -0,0 +1,38 @@ +package utils + +const HOUR_NANO = int64(3600000000000) + +type LogsListTsRange struct { + Start int64 + End int64 +} + +func GetLogsListTsRanges(start, end int64) []LogsListTsRange { + startNano := GetEpochNanoSecs(start) + endNano := GetEpochNanoSecs(end) + result := []LogsListTsRange{} + + if endNano-startNano > HOUR_NANO { + bucket := HOUR_NANO + tStartNano := endNano - bucket + + complete := false + for { + result = append(result, LogsListTsRange{Start: tStartNano, End: endNano}) + if complete { + break + } + + bucket = bucket * 2 + endNano = tStartNano + tStartNano = tStartNano - bucket + + // break condition + if tStartNano <= startNano { + complete = true + tStartNano = startNano + } + } + } + return result +} diff --git a/pkg/query-service/utils/logs_test.go b/pkg/query-service/utils/logs_test.go new file mode 100644 index 0000000000..939fa5fa1b --- /dev/null +++ b/pkg/query-service/utils/logs_test.go @@ -0,0 +1,49 @@ +package utils + +import "testing" + +func TestLogsListTsRange(t *testing.T) { + startEndData := []struct { + name string + start int64 + end int64 + res []LogsListTsRange + }{ + { + name: "testing for less then one hour", + start: 1722262800000000000, // July 29, 2024 7:50:00 PM + end: 1722263800000000000, // July 29, 2024 8:06:40 PM + res: []LogsListTsRange{}, + }, + { + name: "testing for more than one hour", + start: 1722255800000000000, // July 29, 2024 5:53:20 PM + end: 1722262800000000000, // July 29, 2024 8:06:40 PM + res: []LogsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM + }, + }, + { + name: "testing for 1 day", + start: 1722171576000000000, + end: 1722262800000000000, + res: []LogsListTsRange{ + {1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM + {1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM + {1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM + {1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM + {1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM + }, + }, + } + + for _, test := range startEndData { + res := GetLogsListTsRanges(test.start, test.end) + for i, v := range res { + if test.res[i].Start != v.Start || test.res[i].End != v.End { + t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) + } + } + } +}