Skip to content

Commit

Permalink
fix: move getTsRanges to utils
Browse files Browse the repository at this point in the history
  • Loading branch information
@nityanandagohain
nityanandagohain committed Sep 12, 2024
1 parent 803d480 commit 9beb529
Show file tree
Hide file tree
Showing 6 changed files with 91 additions and 169 deletions.
41 changes: 2 additions & 39 deletions pkg/query-service/app/querier/querier.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,44 +470,7 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang
return results, errQueriesByName, err
}

type logsListTsRange struct {
Start int64
End int64
}

const HOUR_NANO = int64(3600000000000)

func getLogsListTsRanges(start, end int64) []logsListTsRange {
startNano := utils.GetEpochNanoSecs(start)
endNano := utils.GetEpochNanoSecs(end)
result := []logsListTsRange{}

if endNano-startNano > HOUR_NANO {
bucket := HOUR_NANO
tStartNano := endNano - bucket

complete := false
for {
result = append(result, logsListTsRange{Start: tStartNano, End: endNano})
if complete {
break
}

bucket = bucket * 2
endNano = tStartNano
tStartNano = tStartNano - bucket

// break condition
if tStartNano <= startNano {
complete = true
tStartNano = startNano
}
}
}
return result
}

func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []logsListTsRange) ([]*v3.Result, map[string]error, error) {
func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) {
res := make([]*v3.Result, 0)
qName := ""
pageSize := uint64(0)
Expand Down Expand Up @@ -572,7 +535,7 @@ func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRan
for _, v := range params.CompositeQuery.BuilderQueries {
// only allow of logs queries with timestamp ordering desc
if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" {
startEndArr := getLogsListTsRanges(params.Start, params.End)
startEndArr := utils.GetLogsListTsRanges(params.Start, params.End)
if len(startEndArr) > 0 {
return q.runLogsListQuery(ctx, params, keys, startEndArr)
}
Expand Down
45 changes: 0 additions & 45 deletions pkg/query-service/app/querier/querier_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1055,48 +1055,3 @@ func TestQueryRangeValueTypePromQL(t *testing.T) {
}
}
}
func TestLogsListTsRange(t *testing.T) {
startEndData := []struct {
name string
start int64
end int64
res []logsListTsRange
}{
{
name: "testing for less then one hour",
start: 1722262800000000000, // July 29, 2024 7:50:00 PM
end: 1722263800000000000, // July 29, 2024 8:06:40 PM
res: []logsListTsRange{},
},
{
name: "testing for more than one hour",
start: 1722255800000000000, // July 29, 2024 5:53:20 PM
end: 1722262800000000000, // July 29, 2024 8:06:40 PM
res: []logsListTsRange{
{1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM
{1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM
},
},
{
name: "testing for 1 day",
start: 1722171576000000000,
end: 1722262800000000000,
res: []logsListTsRange{
{1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM
{1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM
{1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM
{1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM
{1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM
},
},
}

for _, test := range startEndData {
res := getLogsListTsRanges(test.start, test.end)
for i, v := range res {
if test.res[i].Start != v.Start || test.res[i].End != v.End {
t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End)
}
}
}
}
41 changes: 2 additions & 39 deletions pkg/query-service/app/querier/v2/querier.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,44 +479,7 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang
return results, errQueriesByName, err
}

type logsListTsRange struct {
Start int64
End int64
}

const HOUR_NANO = int64(3600000000000)

func getLogsListTsRanges(start, end int64) []logsListTsRange {
startNano := utils.GetEpochNanoSecs(start)
endNano := utils.GetEpochNanoSecs(end)
result := []logsListTsRange{}

if endNano-startNano > HOUR_NANO {
bucket := HOUR_NANO
tStartNano := endNano - bucket

complete := false
for {
result = append(result, logsListTsRange{Start: tStartNano, End: endNano})
if complete {
break
}

bucket = bucket * 2
endNano = tStartNano
tStartNano = tStartNano - bucket

// break condition
if tStartNano <= startNano {
complete = true
tStartNano = startNano
}
}
}
return result
}

func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []logsListTsRange) ([]*v3.Result, map[string]error, error) {
func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) {
res := make([]*v3.Result, 0)
qName := ""
pageSize := uint64(0)
Expand Down Expand Up @@ -581,7 +544,7 @@ func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRan
for _, v := range params.CompositeQuery.BuilderQueries {
// only allow of logs queries with timestamp ordering desc
if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" {
startEndArr := getLogsListTsRanges(params.Start, params.End)
startEndArr := utils.GetLogsListTsRanges(params.Start, params.End)
if len(startEndArr) > 0 {
return q.runLogsListQuery(ctx, params, keys, startEndArr)
}
Expand Down
46 changes: 0 additions & 46 deletions pkg/query-service/app/querier/v2/querier_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1107,49 +1107,3 @@ func TestV2QueryRangeValueTypePromQL(t *testing.T) {
}
}
}

func TestLogsListTsRange(t *testing.T) {
startEndData := []struct {
name string
start int64
end int64
res []logsListTsRange
}{
{
name: "testing for less then one hour",
start: 1722262800000000000, // July 29, 2024 7:50:00 PM
end: 1722263800000000000, // July 29, 2024 8:06:40 PM
res: []logsListTsRange{},
},
{
name: "testing for more than one hour",
start: 1722255800000000000, // July 29, 2024 5:53:20 PM
end: 1722262800000000000, // July 29, 2024 8:06:40 PM
res: []logsListTsRange{
{1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM
{1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM
},
},
{
name: "testing for 1 day",
start: 1722171576000000000,
end: 1722262800000000000,
res: []logsListTsRange{
{1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM
{1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM
{1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM
{1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM
{1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM
},
},
}

for _, test := range startEndData {
res := getLogsListTsRanges(test.start, test.end)
for i, v := range res {
if test.res[i].Start != v.Start || test.res[i].End != v.End {
t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End)
}
}
}
}
38 changes: 38 additions & 0 deletions pkg/query-service/utils/logs.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package utils

const HOUR_NANO = int64(3600000000000)

type LogsListTsRange struct {
Start int64
End int64
}

func GetLogsListTsRanges(start, end int64) []LogsListTsRange {
startNano := GetEpochNanoSecs(start)
endNano := GetEpochNanoSecs(end)
result := []LogsListTsRange{}

if endNano-startNano > HOUR_NANO {
bucket := HOUR_NANO
tStartNano := endNano - bucket

complete := false
for {
result = append(result, LogsListTsRange{Start: tStartNano, End: endNano})
if complete {
break
}

bucket = bucket * 2
endNano = tStartNano
tStartNano = tStartNano - bucket

// break condition
if tStartNano <= startNano {
complete = true
tStartNano = startNano
}
}
}
return result
}
49 changes: 49 additions & 0 deletions pkg/query-service/utils/logs_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package utils

import "testing"

func TestLogsListTsRange(t *testing.T) {
startEndData := []struct {
name string
start int64
end int64
res []LogsListTsRange
}{
{
name: "testing for less then one hour",
start: 1722262800000000000, // July 29, 2024 7:50:00 PM
end: 1722263800000000000, // July 29, 2024 8:06:40 PM
res: []LogsListTsRange{},
},
{
name: "testing for more than one hour",
start: 1722255800000000000, // July 29, 2024 5:53:20 PM
end: 1722262800000000000, // July 29, 2024 8:06:40 PM
res: []LogsListTsRange{
{1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM
{1722255800000000000, 1722259200000000000}, // July 29, 2024 5:53:20 PM - July 29, 2024 6:50:00 PM
},
},
{
name: "testing for 1 day",
start: 1722171576000000000,
end: 1722262800000000000,
res: []LogsListTsRange{
{1722259200000000000, 1722262800000000000}, // July 29, 2024 6:50:00 PM - July 29, 2024 7:50:00 PM
{1722252000000000000, 1722259200000000000}, // July 29, 2024 4:50:00 PM - July 29, 2024 6:50:00 PM
{1722237600000000000, 1722252000000000000}, // July 29, 2024 12:50:00 PM - July 29, 2024 4:50:00 PM
{1722208800000000000, 1722237600000000000}, // July 29, 2024 4:50:00 AM - July 29, 2024 12:50:00 PM
{1722171576000000000, 1722208800000000000}, // July 28, 2024 6:29:36 PM - July 29, 2024 4:50:00 AM
},
},
}

for _, test := range startEndData {
res := GetLogsListTsRanges(test.start, test.end)
for i, v := range res {
if test.res[i].Start != v.Start || test.res[i].End != v.End {
t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End)
}
}
}
}

0 comments on commit 9beb529

Please sign in to comment.