From e6766023ddb67724421587407094778165fafb9d Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Mon, 16 Dec 2024 15:29:16 +0530 Subject: [PATCH 1/6] chore: use tag attributes v2 table (#6616) --- .../clickhouseReader/filter_suggestions.go | 36 ++++------ .../app/clickhouseReader/options.go | 12 ++-- .../app/clickhouseReader/reader.go | 69 +++++++------------ .../integration/filter_suggestions_test.go | 19 +++-- 4 files changed, 51 insertions(+), 85 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/filter_suggestions.go b/pkg/query-service/app/clickhouseReader/filter_suggestions.go index 4de924ddc3..eeda450050 100644 --- a/pkg/query-service/app/clickhouseReader/filter_suggestions.go +++ b/pkg/query-service/app/clickhouseReader/filter_suggestions.go @@ -138,17 +138,17 @@ func (r *ClickHouseReader) getValuesForLogAttributes( ``` select * from ( ( - select tagKey, stringTagValue, int64TagValue, float64TagValue - from signoz_logs.distributed_tag_attributes - where tagKey = $1 and ( - stringTagValue != '' or int64TagValue is not null or float64TagValue is not null + select tag_key, string_value, number_value + from signoz_logs.distributed_tag_attributes_v2 + where tag_key = $1 and ( + string_value != '' or number_value is not null ) limit 2 ) UNION DISTINCT ( - select tagKey, stringTagValue, int64TagValue, float64TagValue - from signoz_logs.distributed_tag_attributes - where tagKey = $2 and ( - stringTagValue != '' or int64TagValue is not null or float64TagValue is not null + select tag_key, string_value, number_value + from signoz_logs.distributed_tag_attributes_v2 + where tag_key = $2 and ( + string_value != '' or number_value is not null ) limit 2 ) @@ -156,9 +156,6 @@ func (r *ClickHouseReader) getValuesForLogAttributes( ``` Since tag_attributes table uses ReplacingMergeTree, the values would be distinct and no order by is being used to ensure the `limit` clause minimizes the amount of data scanned. - - This query scanned ~30k rows per attribute on fiscalnote-v2 for attributes like `message` and `time` - that had >~110M values each */ if len(attributes) > 10 { @@ -173,13 +170,13 @@ func (r *ClickHouseReader) getValuesForLogAttributes( tagKeyQueryArgs := []any{} for idx, attrib := range attributes { tagQueries = append(tagQueries, fmt.Sprintf(`( - select tagKey, stringTagValue, int64TagValue, float64TagValue + select tag_key, string_value, number_value from %s.%s - where tagKey = $%d and ( - stringTagValue != '' or int64TagValue is not null or float64TagValue is not null + where tag_key = $%d and ( + string_value != '' or number_value is not null ) limit %d - )`, r.logsDB, r.logsTagAttributeTable, idx+1, limit)) + )`, r.logsDB, r.logsTagAttributeTableV2, idx+1, limit)) tagKeyQueryArgs = append(tagKeyQueryArgs, attrib.Key) } @@ -211,10 +208,9 @@ func (r *ClickHouseReader) getValuesForLogAttributes( var tagKey string var stringValue string var float64Value sql.NullFloat64 - var int64Value sql.NullInt64 err := rows.Scan( - &tagKey, &stringValue, &int64Value, &float64Value, + &tagKey, &stringValue, &float64Value, ) if err != nil { return nil, model.InternalError(fmt.Errorf( @@ -228,12 +224,6 @@ func (r *ClickHouseReader) getValuesForLogAttributes( result[attrResultIdx] = append(result[attrResultIdx], stringValue) } - } else if int64Value.Valid { - attrResultIdx := resultIdxForAttrib(tagKey, v3.AttributeKeyDataTypeInt64) - if attrResultIdx >= 0 { - result[attrResultIdx] = append(result[attrResultIdx], int64Value.Int64) - } - } else if float64Value.Valid { attrResultIdx := resultIdxForAttrib(tagKey, v3.AttributeKeyDataTypeFloat64) if attrResultIdx >= 0 { diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 25eea0c7ff..b9de1db054 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -29,14 +29,14 @@ const ( defaultSpansTable string = "distributed_signoz_spans" defaultDependencyGraphTable string = "distributed_dependency_graph_minutes_v2" defaultTopLevelOperationsTable string = "distributed_top_level_operations" - defaultSpanAttributeTable string = "distributed_span_attributes" + defaultSpanAttributeTableV2 string = "distributed_tag_attributes_v2" defaultSpanAttributeKeysTable string = "distributed_span_attributes_keys" defaultLogsDB string = "signoz_logs" defaultLogsTable string = "distributed_logs" defaultLogsLocalTable string = "logs" defaultLogAttributeKeysTable string = "distributed_logs_attribute_keys" defaultLogResourceKeysTable string = "distributed_logs_resource_keys" - defaultLogTagAttributeTable string = "distributed_tag_attributes" + defaultLogTagAttributeTableV2 string = "distributed_tag_attributes_v2" defaultLiveTailRefreshSeconds int = 5 defaultWriteBatchDelay time.Duration = 5 * time.Second defaultWriteBatchSize int = 10000 @@ -69,7 +69,7 @@ type namespaceConfig struct { UsageExplorerTable string SpansTable string ErrorTable string - SpanAttributeTable string + SpanAttributeTableV2 string SpanAttributeKeysTable string DependencyGraphTable string TopLevelOperationsTable string @@ -78,7 +78,7 @@ type namespaceConfig struct { LogsLocalTable string LogsAttributeKeysTable string LogsResourceKeysTable string - LogsTagAttributeTable string + LogsTagAttributeTableV2 string LiveTailRefreshSeconds int WriteBatchDelay time.Duration WriteBatchSize int @@ -167,7 +167,7 @@ func NewOptions( DurationTable: defaultDurationTable, UsageExplorerTable: defaultUsageExplorerTable, SpansTable: defaultSpansTable, - SpanAttributeTable: defaultSpanAttributeTable, + SpanAttributeTableV2: defaultSpanAttributeTableV2, SpanAttributeKeysTable: defaultSpanAttributeKeysTable, DependencyGraphTable: defaultDependencyGraphTable, TopLevelOperationsTable: defaultTopLevelOperationsTable, @@ -176,7 +176,7 @@ func NewOptions( LogsLocalTable: defaultLogsLocalTable, LogsAttributeKeysTable: defaultLogAttributeKeysTable, LogsResourceKeysTable: defaultLogResourceKeysTable, - LogsTagAttributeTable: defaultLogTagAttributeTable, + LogsTagAttributeTableV2: defaultLogTagAttributeTableV2, LiveTailRefreshSeconds: defaultLiveTailRefreshSeconds, WriteBatchDelay: defaultWriteBatchDelay, WriteBatchSize: defaultWriteBatchSize, diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 4f1aa99dfc..9c7828af45 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -118,7 +118,7 @@ type ClickHouseReader struct { errorTable string usageExplorerTable string SpansTable string - spanAttributeTable string + spanAttributeTableV2 string spanAttributesKeysTable string dependencyGraphTable string topLevelOperationsTable string @@ -127,7 +127,7 @@ type ClickHouseReader struct { logsLocalTable string logsAttributeKeys string logsResourceKeys string - logsTagAttributeTable string + logsTagAttributeTableV2 string queryEngine *promql.Engine remoteStorage *remote.Storage fanoutStorage *storage.Storage @@ -246,7 +246,7 @@ func NewReaderFromClickhouseConnection( usageExplorerTable: options.primary.UsageExplorerTable, durationTable: options.primary.DurationTable, SpansTable: options.primary.SpansTable, - spanAttributeTable: options.primary.SpanAttributeTable, + spanAttributeTableV2: options.primary.SpanAttributeTableV2, spanAttributesKeysTable: options.primary.SpanAttributeKeysTable, dependencyGraphTable: options.primary.DependencyGraphTable, topLevelOperationsTable: options.primary.TopLevelOperationsTable, @@ -255,7 +255,7 @@ func NewReaderFromClickhouseConnection( logsLocalTable: options.primary.LogsLocalTable, logsAttributeKeys: options.primary.LogsAttributeKeysTable, logsResourceKeys: options.primary.LogsResourceKeysTable, - logsTagAttributeTable: options.primary.LogsTagAttributeTable, + logsTagAttributeTableV2: options.primary.LogsTagAttributeTableV2, liveTailRefreshSeconds: options.primary.LiveTailRefreshSeconds, promConfigFile: configFile, featureFlags: featureFlag, @@ -1035,29 +1035,6 @@ func addExistsOperator(item model.TagQuery, tagMapType string, not bool) (string return fmt.Sprintf(" AND %s (%s)", notStr, strings.Join(tagOperatorPair, " OR ")), args } -func excludeTags(_ context.Context, tags []string) []string { - excludedTagsMap := map[string]bool{ - "http.code": true, - "http.route": true, - "http.method": true, - "http.url": true, - "http.status_code": true, - "http.host": true, - "messaging.system": true, - "messaging.operation": true, - "error": true, - "service.name": true, - } - newTags := make([]string, 0) - for _, tag := range tags { - _, ok := excludedTagsMap[tag] - if !ok { - newTags = append(newTags, tag) - } - } - return newTags -} - func (r *ClickHouseReader) GetTopOperationsV2(ctx context.Context, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError) { namedArgs := []interface{}{ @@ -3503,7 +3480,7 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v case v3.AggregateOperatorCountDistinct, v3.AggregateOperatorCount: - where = "tagKey ILIKE $1" + where = "tag_key ILIKE $1" stringAllowed = true case v3.AggregateOperatorRateSum, @@ -3524,7 +3501,7 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax: - where = "tagKey ILIKE $1 AND (tagDataType='int64' or tagDataType='float64')" + where = "tag_key ILIKE $1 AND (tag_data_type='int64' or tag_data_type='float64')" stringAllowed = false case v3.AggregateOperatorNoOp: @@ -3533,7 +3510,7 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v return nil, fmt.Errorf("unsupported aggregate operator") } - query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, tagDataType from %s.%s WHERE %s limit $2", r.logsDB, r.logsTagAttributeTable, where) + query = fmt.Sprintf("SELECT DISTINCT(tag_key), tag_type, tag_data_type from %s.%s WHERE %s limit $2", r.logsDB, r.logsTagAttributeTableV2, where) rows, err = r.db.Query(ctx, query, fmt.Sprintf("%%%s%%", req.SearchText), req.Limit) if err != nil { zap.L().Error("Error while executing query", zap.Error(err)) @@ -3582,10 +3559,10 @@ func (r *ClickHouseReader) GetLogAttributeKeys(ctx context.Context, req *v3.Filt var response v3.FilterAttributeKeyResponse if len(req.SearchText) != 0 { - query = fmt.Sprintf("select distinct tagKey, tagType, tagDataType from %s.%s where tagKey ILIKE $1 limit $2", r.logsDB, r.logsTagAttributeTable) + query = fmt.Sprintf("select distinct tag_key, tag_type, tag_data_type from %s.%s where tag_key ILIKE $1 limit $2", r.logsDB, r.logsTagAttributeTableV2) rows, err = r.db.Query(ctx, query, fmt.Sprintf("%%%s%%", req.SearchText), req.Limit) } else { - query = fmt.Sprintf("select distinct tagKey, tagType, tagDataType from %s.%s limit $1", r.logsDB, r.logsTagAttributeTable) + query = fmt.Sprintf("select distinct tag_key, tag_type, tag_data_type from %s.%s limit $1", r.logsDB, r.logsTagAttributeTableV2) rows, err = r.db.Query(ctx, query, req.Limit) } @@ -3662,11 +3639,11 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi query := "select distinct" switch req.FilterAttributeKeyDataType { case v3.AttributeKeyDataTypeInt64: - filterValueColumn = "int64TagValue" + filterValueColumn = "number_value" case v3.AttributeKeyDataTypeFloat64: - filterValueColumn = "float64TagValue" + filterValueColumn = "number_value" case v3.AttributeKeyDataTypeString: - filterValueColumn = "stringTagValue" + filterValueColumn = "string_value" } searchText := fmt.Sprintf("%%%s%%", req.SearchText) @@ -3694,10 +3671,10 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi if req.FilterAttributeKeyDataType != v3.AttributeKeyDataTypeString { filterValueColumnWhere = fmt.Sprintf("toString(%s)", filterValueColumn) } - query = fmt.Sprintf("select distinct %s from %s.%s where tagKey=$1 and %s ILIKE $2 and tagType=$3 limit $4", filterValueColumn, r.logsDB, r.logsTagAttributeTable, filterValueColumnWhere) + query = fmt.Sprintf("SELECT DISTINCT %s FROM %s.%s WHERE tag_key=$1 AND %s ILIKE $2 AND tag_type=$3 LIMIT $4", filterValueColumn, r.logsDB, r.logsTagAttributeTableV2, filterValueColumnWhere) rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, searchText, req.TagType, req.Limit) } else { - query = fmt.Sprintf("select distinct %s from %s.%s where tagKey=$1 and tagType=$2 limit $3", filterValueColumn, r.logsDB, r.logsTagAttributeTable) + query = fmt.Sprintf("SELECT DISTINCT %s FROM %s.%s WHERE tag_key=$1 AND tag_type=$2 LIMIT $3", filterValueColumn, r.logsDB, r.logsTagAttributeTableV2) rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, req.TagType, req.Limit) } @@ -4162,7 +4139,7 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req case v3.AggregateOperatorCountDistinct, v3.AggregateOperatorCount: - where = "tagKey ILIKE $1" + where = "tag_key ILIKE $1" stringAllowed = true case v3.AggregateOperatorRateSum, @@ -4183,7 +4160,7 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax: - where = "tagKey ILIKE $1 AND dataType='float64'" + where = "tag_key ILIKE $1 AND tag_data_type='float64'" stringAllowed = false case v3.AggregateOperatorNoOp: @@ -4191,7 +4168,7 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req default: return nil, fmt.Errorf("unsupported aggregate operator") } - query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType FROM %s.%s WHERE %s", r.TraceDB, r.spanAttributeTable, where) + query = fmt.Sprintf("SELECT DISTINCT(tag_key), tag_type, tag_data_type FROM %s.%s WHERE %s", r.TraceDB, r.spanAttributeTableV2, where) if req.Limit != 0 { query = query + fmt.Sprintf(" LIMIT %d;", req.Limit) } @@ -4253,7 +4230,7 @@ func (r *ClickHouseReader) GetTraceAttributeKeys(ctx context.Context, req *v3.Fi var rows driver.Rows var response v3.FilterAttributeKeyResponse - query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType FROM %s.%s WHERE tagKey ILIKE $1 LIMIT $2", r.TraceDB, r.spanAttributeTable) + query = fmt.Sprintf("SELECT DISTINCT(tag_key), tag_type, tag_data_type FROM %s.%s WHERE tag_key ILIKE $1 LIMIT $2", r.TraceDB, r.spanAttributeTableV2) rows, err = r.db.Query(ctx, query, fmt.Sprintf("%%%s%%", req.SearchText), req.Limit) @@ -4335,12 +4312,12 @@ func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3. }, nil } - query = "select distinct" + query = "SELECT DISTINCT" switch req.FilterAttributeKeyDataType { case v3.AttributeKeyDataTypeFloat64: - filterValueColumn = "float64TagValue" + filterValueColumn = "number_value" case v3.AttributeKeyDataTypeString: - filterValueColumn = "stringTagValue" + filterValueColumn = "string_value" } searchText := fmt.Sprintf("%%%s%%", req.SearchText) @@ -4361,14 +4338,14 @@ func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3. if r.useTraceNewSchema { where += " AND ts_bucket_start >= toUInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR))" } - query = fmt.Sprintf("select distinct %s from %s.%s where %s and %s ILIKE $1 limit $2", selectKey, r.TraceDB, r.traceTableName, where, filterValueColumnWhere) + query = fmt.Sprintf("SELECT DISTINCT %s FROM %s.%s WHERE %s AND %s ILIKE $1 LIMIT $2", selectKey, r.TraceDB, r.traceTableName, where, filterValueColumnWhere) rows, err = r.db.Query(ctx, query, searchText, req.Limit) } else { filterValueColumnWhere := filterValueColumn if req.FilterAttributeKeyDataType != v3.AttributeKeyDataTypeString { filterValueColumnWhere = fmt.Sprintf("toString(%s)", filterValueColumn) } - query = fmt.Sprintf("select distinct %s from %s.%s where tagKey=$1 and %s ILIKE $2 and tagType=$3 limit $4", filterValueColumn, r.TraceDB, r.spanAttributeTable, filterValueColumnWhere) + query = fmt.Sprintf("SELECT DISTINCT %s FROM %s.%s WHERE tag_key=$1 AND %s ILIKE $2 AND tag_type=$3 LIMIT $4", filterValueColumn, r.TraceDB, r.spanAttributeTableV2, filterValueColumnWhere) rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, searchText, req.TagType, req.Limit) } diff --git a/pkg/query-service/tests/integration/filter_suggestions_test.go b/pkg/query-service/tests/integration/filter_suggestions_test.go index 6c8224be50..793ca7d442 100644 --- a/pkg/query-service/tests/integration/filter_suggestions_test.go +++ b/pkg/query-service/tests/integration/filter_suggestions_test.go @@ -199,9 +199,9 @@ func (tb *FilterSuggestionsTestBed) mockAttribKeysQueryResponse( attribsToReturn []v3.AttributeKey, ) { cols := []mockhouse.ColumnType{} - cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tagKey"}) - cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tagType"}) - cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tagDataType"}) + cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tag_key"}) + cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tag_type"}) + cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tag_data_type"}) values := [][]any{} for _, a := range attribsToReturn { @@ -213,7 +213,7 @@ func (tb *FilterSuggestionsTestBed) mockAttribKeysQueryResponse( } tb.mockClickhouse.ExpectQuery( - "select.*from.*signoz_logs.distributed_tag_attributes.*", + "select.*from.*signoz_logs.distributed_tag_attributes_v2.*", ).WithArgs( constants.DefaultFilterSuggestionsAttributesLimit, ).WillReturnRows( @@ -236,10 +236,9 @@ func (tb *FilterSuggestionsTestBed) mockAttribValuesQueryResponse( stringValuesToReturn [][]string, ) { resultCols := []mockhouse.ColumnType{ - {Type: "String", Name: "tagKey"}, - {Type: "String", Name: "stringTagValue"}, - {Type: "Nullable(Int64)", Name: "int64TagValue"}, - {Type: "Nullable(Float64)", Name: "float64TagValue"}, + {Type: "String", Name: "tag_key"}, + {Type: "String", Name: "string_value"}, + {Type: "Nullable(Int64)", Name: "number_value"}, } expectedAttribKeysInQuery := []any{} @@ -248,13 +247,13 @@ func (tb *FilterSuggestionsTestBed) mockAttribValuesQueryResponse( expectedAttribKeysInQuery = append(expectedAttribKeysInQuery, attrib.Key) for _, stringTagValue := range stringValuesToReturn[idx] { mockResultRows = append(mockResultRows, []any{ - attrib.Key, stringTagValue, nil, nil, + attrib.Key, stringTagValue, nil, }) } } tb.mockClickhouse.ExpectQuery( - "select.*tagKey.*stringTagValue.*int64TagValue.*float64TagValue.*distributed_tag_attributes.*tagKey", + "select.*tag_key.*string_value.*number_value.*distributed_tag_attributes_v2.*tag_key", ).WithArgs(expectedAttribKeysInQuery...).WillReturnRows(mockhouse.NewRows(resultCols, mockResultRows)) } From 951593b0a3d5d9434b629447955b9c16995a92e5 Mon Sep 17 00:00:00 2001 From: Vibhu Pandey Date: Mon, 16 Dec 2024 15:53:23 +0530 Subject: [PATCH 2/6] feat(licenses): deprecate licenses v2 (#6626) deprecate licenses v2 --- ee/query-service/app/api/api.go | 2 - ee/query-service/app/api/license.go | 31 +-- ee/query-service/app/server.go | 4 +- .../integrations/signozio/response.go | 12 - .../integrations/signozio/signozio.go | 81 ------- ee/query-service/license/db.go | 32 +-- ee/query-service/license/manager.go | 227 +----------------- ee/query-service/main.go | 3 - pkg/query-service/app/http_handler.go | 4 - 9 files changed, 22 insertions(+), 374 deletions(-) diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index 5d7d6d2ffa..181186d323 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -41,7 +41,6 @@ type APIHandlerOptions struct { FluxInterval time.Duration UseLogsNewSchema bool UseTraceNewSchema bool - UseLicensesV3 bool } type APIHandler struct { @@ -68,7 +67,6 @@ func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) { FluxInterval: opts.FluxInterval, UseLogsNewSchema: opts.UseLogsNewSchema, UseTraceNewSchema: opts.UseTraceNewSchema, - UseLicensesV3: opts.UseLicensesV3, }) if err != nil { diff --git a/ee/query-service/app/api/license.go b/ee/query-service/app/api/license.go index 7138e29f80..7a098d4e63 100644 --- a/ee/query-service/app/api/license.go +++ b/ee/query-service/app/api/license.go @@ -84,13 +84,6 @@ func (ah *APIHandler) listLicenses(w http.ResponseWriter, r *http.Request) { } func (ah *APIHandler) applyLicense(w http.ResponseWriter, r *http.Request) { - if ah.UseLicensesV3 { - // if the licenses v3 is toggled on then do not apply license in v2 and run the validator! - // TODO: remove after migration to v3 and deprecation from zeus - zap.L().Info("early return from apply license v2 call") - render.Success(w, http.StatusOK, nil) - return - } var l model.License if err := json.NewDecoder(r.Body).Decode(&l); err != nil { @@ -102,7 +95,7 @@ func (ah *APIHandler) applyLicense(w http.ResponseWriter, r *http.Request) { RespondError(w, model.BadRequest(fmt.Errorf("license key is required")), nil) return } - license, apiError := ah.LM().Activate(r.Context(), l.Key) + license, apiError := ah.LM().ActivateV3(r.Context(), l.Key) if apiError != nil { RespondError(w, apiError, nil) return @@ -265,24 +258,12 @@ func convertLicenseV3ToLicenseV2(licenses []*model.LicenseV3) []model.License { } func (ah *APIHandler) listLicensesV2(w http.ResponseWriter, r *http.Request) { - - var licenses []model.License - - if ah.UseLicensesV3 { - licensesV3, err := ah.LM().GetLicensesV3(r.Context()) - if err != nil { - RespondError(w, err, nil) - return - } - licenses = convertLicenseV3ToLicenseV2(licensesV3) - } else { - _licenses, apiError := ah.LM().GetLicenses(r.Context()) - if apiError != nil { - RespondError(w, apiError, nil) - return - } - licenses = _licenses + licensesV3, apierr := ah.LM().GetLicensesV3(r.Context()) + if apierr != nil { + RespondError(w, apierr, nil) + return } + licenses := convertLicenseV3ToLicenseV2(licensesV3) resp := model.Licenses{ TrialStart: -1, diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index a970a34b0e..938b72b5a3 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -78,7 +78,6 @@ type ServerOptions struct { GatewayUrl string UseLogsNewSchema bool UseTraceNewSchema bool - UseLicensesV3 bool } // Server runs HTTP api service @@ -135,7 +134,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } // initiate license manager - lm, err := licensepkg.StartManager("sqlite", localDB, serverOptions.UseLicensesV3) + lm, err := licensepkg.StartManager("sqlite", localDB) if err != nil { return nil, err } @@ -274,7 +273,6 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { Gateway: gatewayProxy, UseLogsNewSchema: serverOptions.UseLogsNewSchema, UseTraceNewSchema: serverOptions.UseTraceNewSchema, - UseLicensesV3: serverOptions.UseLicensesV3, } apiHandler, err := api.NewAPIHandler(apiOpts) diff --git a/ee/query-service/integrations/signozio/response.go b/ee/query-service/integrations/signozio/response.go index f0b0132d1b..891ea77da1 100644 --- a/ee/query-service/integrations/signozio/response.go +++ b/ee/query-service/integrations/signozio/response.go @@ -2,18 +2,6 @@ package signozio type status string -type ActivationResult struct { - Status status `json:"status"` - Data *ActivationResponse `json:"data,omitempty"` - ErrorType string `json:"errorType,omitempty"` - Error string `json:"error,omitempty"` -} - -type ActivationResponse struct { - ActivationId string `json:"ActivationId"` - PlanDetails string `json:"PlanDetails"` -} - type ValidateLicenseResponse struct { Status status `json:"status"` Data map[string]interface{} `json:"data"` diff --git a/ee/query-service/integrations/signozio/signozio.go b/ee/query-service/integrations/signozio/signozio.go index 6c0b937c80..a3a5cad414 100644 --- a/ee/query-service/integrations/signozio/signozio.go +++ b/ee/query-service/integrations/signozio/signozio.go @@ -10,7 +10,6 @@ import ( "time" "github.com/pkg/errors" - "go.uber.org/zap" "go.signoz.io/signoz/ee/query-service/constants" "go.signoz.io/signoz/ee/query-service/model" @@ -39,86 +38,6 @@ func init() { C = New() } -// ActivateLicense sends key to license.signoz.io and gets activation data -func ActivateLicense(key, siteId string) (*ActivationResponse, *model.ApiError) { - licenseReq := map[string]string{ - "key": key, - "siteId": siteId, - } - - reqString, _ := json.Marshal(licenseReq) - httpResponse, err := http.Post(C.Prefix+"/licenses/activate", APPLICATION_JSON, bytes.NewBuffer(reqString)) - - if err != nil { - zap.L().Error("failed to connect to license.signoz.io", zap.Error(err)) - return nil, model.BadRequest(fmt.Errorf("unable to connect with license.signoz.io, please check your network connection")) - } - - httpBody, err := io.ReadAll(httpResponse.Body) - if err != nil { - zap.L().Error("failed to read activation response from license.signoz.io", zap.Error(err)) - return nil, model.BadRequest(fmt.Errorf("failed to read activation response from license.signoz.io")) - } - - defer httpResponse.Body.Close() - - // read api request result - result := ActivationResult{} - err = json.Unmarshal(httpBody, &result) - if err != nil { - zap.L().Error("failed to marshal activation response from license.signoz.io", zap.Error(err)) - return nil, model.InternalError(errors.Wrap(err, "failed to marshal license activation response")) - } - - switch httpResponse.StatusCode { - case 200, 201: - return result.Data, nil - case 400, 401: - return nil, model.BadRequest(fmt.Errorf(fmt.Sprintf("failed to activate: %s", result.Error))) - default: - return nil, model.InternalError(fmt.Errorf(fmt.Sprintf("failed to activate: %s", result.Error))) - } - -} - -// ValidateLicense validates the license key -func ValidateLicense(activationId string) (*ActivationResponse, *model.ApiError) { - validReq := map[string]string{ - "activationId": activationId, - } - - reqString, _ := json.Marshal(validReq) - response, err := http.Post(C.Prefix+"/licenses/validate", APPLICATION_JSON, bytes.NewBuffer(reqString)) - - if err != nil { - return nil, model.BadRequest(errors.Wrap(err, "unable to connect with license.signoz.io, please check your network connection")) - } - - body, err := io.ReadAll(response.Body) - if err != nil { - return nil, model.BadRequest(errors.Wrap(err, "failed to read validation response from license.signoz.io")) - } - - defer response.Body.Close() - - switch response.StatusCode { - case 200, 201: - a := ActivationResult{} - err = json.Unmarshal(body, &a) - if err != nil { - return nil, model.BadRequest(errors.Wrap(err, "failed to marshal license validation response")) - } - return a.Data, nil - case 400, 401: - return nil, model.BadRequest(errors.Wrap(fmt.Errorf(string(body)), - "bad request error received from license.signoz.io")) - default: - return nil, model.InternalError(errors.Wrap(fmt.Errorf(string(body)), - "internal error received from license.signoz.io")) - } - -} - func ValidateLicenseV3(licenseKey string) (*model.LicenseV3, *model.ApiError) { // Creating an HTTP client with a timeout for better control diff --git a/ee/query-service/license/db.go b/ee/query-service/license/db.go index 4bd34e232c..1dba4053d7 100644 --- a/ee/query-service/license/db.go +++ b/ee/query-service/license/db.go @@ -18,15 +18,13 @@ import ( // Repo is license repo. stores license keys in a secured DB type Repo struct { - db *sqlx.DB - useLicensesV3 bool + db *sqlx.DB } // NewLicenseRepo initiates a new license repo -func NewLicenseRepo(db *sqlx.DB, useLicensesV3 bool) Repo { +func NewLicenseRepo(db *sqlx.DB) Repo { return Repo{ - db: db, - useLicensesV3: useLicensesV3, + db: db, } } @@ -112,26 +110,16 @@ func (r *Repo) GetActiveLicenseV2(ctx context.Context) (*model.License, *basemod // GetActiveLicense fetches the latest active license from DB. // If the license is not present, expect a nil license and a nil error in the output. func (r *Repo) GetActiveLicense(ctx context.Context) (*model.License, *basemodel.ApiError) { - if r.useLicensesV3 { - zap.L().Info("Using licenses v3 for GetActiveLicense") - activeLicenseV3, err := r.GetActiveLicenseV3(ctx) - if err != nil { - return nil, basemodel.InternalError(fmt.Errorf("failed to get active licenses from db: %v", err)) - } - - if activeLicenseV3 == nil { - return nil, nil - } - activeLicenseV2 := model.ConvertLicenseV3ToLicenseV2(activeLicenseV3) - return activeLicenseV2, nil - + activeLicenseV3, err := r.GetActiveLicenseV3(ctx) + if err != nil { + return nil, basemodel.InternalError(fmt.Errorf("failed to get active licenses from db: %v", err)) } - active, err := r.GetActiveLicenseV2(ctx) - if err != nil { - return nil, err + if activeLicenseV3 == nil { + return nil, nil } - return active, nil + activeLicenseV2 := model.ConvertLicenseV3ToLicenseV2(activeLicenseV3) + return activeLicenseV2, nil } func (r *Repo) GetActiveLicenseV3(ctx context.Context) (*model.LicenseV3, error) { diff --git a/ee/query-service/license/manager.go b/ee/query-service/license/manager.go index 0a4370de3f..c036a01ab5 100644 --- a/ee/query-service/license/manager.go +++ b/ee/query-service/license/manager.go @@ -51,12 +51,12 @@ type Manager struct { activeFeatures basemodel.FeatureSet } -func StartManager(dbType string, db *sqlx.DB, useLicensesV3 bool, features ...basemodel.Feature) (*Manager, error) { +func StartManager(dbType string, db *sqlx.DB, features ...basemodel.Feature) (*Manager, error) { if LM != nil { return LM, nil } - repo := NewLicenseRepo(db, useLicensesV3) + repo := NewLicenseRepo(db) err := repo.InitDB(dbType) if err != nil { @@ -67,32 +67,7 @@ func StartManager(dbType string, db *sqlx.DB, useLicensesV3 bool, features ...ba repo: &repo, } - if useLicensesV3 { - // get active license from the db - active, err := m.repo.GetActiveLicenseV2(context.Background()) - if err != nil { - return m, err - } - - // if we have an active license then need to fetch the complete details - if active != nil { - // fetch the new license structure from control plane - licenseV3, apiError := validate.ValidateLicenseV3(active.Key) - if apiError != nil { - return m, apiError - } - - // insert the licenseV3 in sqlite db - apiError = m.repo.InsertLicenseV3(context.Background(), licenseV3) - // if the license already exists move ahead. - if apiError != nil && apiError.Typ != model.ErrorConflict { - return m, apiError - } - zap.L().Info("Successfully inserted license from v2 to v3 table") - } - } - - if err := m.start(useLicensesV3, features...); err != nil { + if err := m.start(features...); err != nil { return m, err } LM = m @@ -100,16 +75,8 @@ func StartManager(dbType string, db *sqlx.DB, useLicensesV3 bool, features ...ba } // start loads active license in memory and initiates validator -func (lm *Manager) start(useLicensesV3 bool, features ...basemodel.Feature) error { - - var err error - if useLicensesV3 { - err = lm.LoadActiveLicenseV3(features...) - } else { - err = lm.LoadActiveLicense(features...) - } - - return err +func (lm *Manager) start(features ...basemodel.Feature) error { + return lm.LoadActiveLicenseV3(features...) } func (lm *Manager) Stop() { @@ -117,31 +84,6 @@ func (lm *Manager) Stop() { <-lm.terminated } -func (lm *Manager) SetActive(l *model.License, features ...basemodel.Feature) { - lm.mutex.Lock() - defer lm.mutex.Unlock() - - if l == nil { - return - } - - lm.activeLicense = l - lm.activeFeatures = append(l.FeatureSet, features...) - // set default features - setDefaultFeatures(lm) - - err := lm.InitFeatures(lm.activeFeatures) - if err != nil { - zap.L().Panic("Couldn't activate features", zap.Error(err)) - } - if !lm.validatorRunning { - // we want to make sure only one validator runs, - // we already have lock() so good to go - lm.validatorRunning = true - go lm.Validator(context.Background()) - } - -} func (lm *Manager) SetActiveV3(l *model.LicenseV3, features ...basemodel.Feature) { lm.mutex.Lock() defer lm.mutex.Unlock() @@ -172,29 +114,6 @@ func setDefaultFeatures(lm *Manager) { lm.activeFeatures = append(lm.activeFeatures, baseconstants.DEFAULT_FEATURE_SET...) } -// LoadActiveLicense loads the most recent active license -func (lm *Manager) LoadActiveLicense(features ...basemodel.Feature) error { - active, err := lm.repo.GetActiveLicense(context.Background()) - if err != nil { - return err - } - if active != nil { - lm.SetActive(active, features...) - } else { - zap.L().Info("No active license found, defaulting to basic plan") - // if no active license is found, we default to basic(free) plan with all default features - lm.activeFeatures = model.BasicPlan - setDefaultFeatures(lm) - err := lm.InitFeatures(lm.activeFeatures) - if err != nil { - zap.L().Error("Couldn't initialize features", zap.Error(err)) - return err - } - } - - return nil -} - func (lm *Manager) LoadActiveLicenseV3(features ...basemodel.Feature) error { active, err := lm.repo.GetActiveLicenseV3(context.Background()) if err != nil { @@ -265,31 +184,6 @@ func (lm *Manager) GetLicensesV3(ctx context.Context) (response []*model.License return response, nil } -// Validator validates license after an epoch of time -func (lm *Manager) Validator(ctx context.Context) { - zap.L().Info("Validator started!") - defer close(lm.terminated) - tick := time.NewTicker(validationFrequency) - defer tick.Stop() - - lm.Validate(ctx) - - for { - select { - case <-lm.done: - return - default: - select { - case <-lm.done: - return - case <-tick.C: - lm.Validate(ctx) - } - } - - } -} - // Validator validates license after an epoch of time func (lm *Manager) ValidatorV3(ctx context.Context) { zap.L().Info("ValidatorV3 started!") @@ -315,73 +209,6 @@ func (lm *Manager) ValidatorV3(ctx context.Context) { } } -// Validate validates the current active license -func (lm *Manager) Validate(ctx context.Context) (reterr error) { - zap.L().Info("License validation started") - if lm.activeLicense == nil { - return nil - } - - defer func() { - lm.mutex.Lock() - - lm.lastValidated = time.Now().Unix() - if reterr != nil { - zap.L().Error("License validation completed with error", zap.Error(reterr)) - atomic.AddUint64(&lm.failedAttempts, 1) - telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_LICENSE_CHECK_FAILED, - map[string]interface{}{"err": reterr.Error()}, "", true, false) - } else { - zap.L().Info("License validation completed with no errors") - } - - lm.mutex.Unlock() - }() - - response, apiError := validate.ValidateLicense(lm.activeLicense.ActivationId) - if apiError != nil { - zap.L().Error("failed to validate license", zap.Error(apiError.Err)) - return apiError.Err - } - - if response.PlanDetails == lm.activeLicense.PlanDetails { - // license plan hasnt changed, nothing to do - return nil - } - - if response.PlanDetails != "" { - - // copy and replace the active license record - l := model.License{ - Key: lm.activeLicense.Key, - CreatedAt: lm.activeLicense.CreatedAt, - PlanDetails: response.PlanDetails, - ValidationMessage: lm.activeLicense.ValidationMessage, - ActivationId: lm.activeLicense.ActivationId, - } - - if err := l.ParsePlan(); err != nil { - zap.L().Error("failed to parse updated license", zap.Error(err)) - return err - } - - // updated plan is parsable, check if plan has changed - if lm.activeLicense.PlanDetails != response.PlanDetails { - err := lm.repo.UpdatePlanDetails(ctx, lm.activeLicense.Key, response.PlanDetails) - if err != nil { - // unexpected db write issue but we can let the user continue - // and wait for update to work in next cycle. - zap.L().Error("failed to validate license", zap.Error(err)) - } - } - - // activate the update license plan - lm.SetActive(&l) - } - - return nil -} - func (lm *Manager) RefreshLicense(ctx context.Context) *model.ApiError { license, apiError := validate.ValidateLicenseV3(lm.activeLicenseV3.Key) @@ -429,50 +256,6 @@ func (lm *Manager) ValidateV3(ctx context.Context) (reterr error) { return nil } -// Activate activates a license key with signoz server -func (lm *Manager) Activate(ctx context.Context, key string) (licenseResponse *model.License, errResponse *model.ApiError) { - defer func() { - if errResponse != nil { - userEmail, err := auth.GetEmailFromJwt(ctx) - if err == nil { - telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_LICENSE_ACT_FAILED, - map[string]interface{}{"err": errResponse.Err.Error()}, userEmail, true, false) - } - } - }() - - response, apiError := validate.ActivateLicense(key, "") - if apiError != nil { - zap.L().Error("failed to activate license", zap.Error(apiError.Err)) - return nil, apiError - } - - l := &model.License{ - Key: key, - ActivationId: response.ActivationId, - PlanDetails: response.PlanDetails, - } - - // parse validity and features from the plan details - err := l.ParsePlan() - - if err != nil { - zap.L().Error("failed to activate license", zap.Error(err)) - return nil, model.InternalError(err) - } - - // store the license before activating it - err = lm.repo.InsertLicense(ctx, l) - if err != nil { - zap.L().Error("failed to activate license", zap.Error(err)) - return nil, model.InternalError(err) - } - - // license is valid, activate it - lm.SetActive(l) - return l, nil -} - func (lm *Manager) ActivateV3(ctx context.Context, licenseKey string) (licenseResponse *model.LicenseV3, errResponse *model.ApiError) { defer func() { if errResponse != nil { diff --git a/ee/query-service/main.go b/ee/query-service/main.go index 23824bd636..dd52ab73a5 100644 --- a/ee/query-service/main.go +++ b/ee/query-service/main.go @@ -95,7 +95,6 @@ func main() { var useLogsNewSchema bool var useTraceNewSchema bool - var useLicensesV3 bool var cacheConfigPath, fluxInterval string var enableQueryServiceLogOTLPExport bool var preferSpanMetrics bool @@ -107,7 +106,6 @@ func main() { flag.BoolVar(&useLogsNewSchema, "use-logs-new-schema", false, "use logs_v2 schema for logs") flag.BoolVar(&useTraceNewSchema, "use-trace-new-schema", false, "use new schema for traces") - flag.BoolVar(&useLicensesV3, "use-licenses-v3", false, "use licenses_v3 schema for licenses") flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.StringVar(&skipTopLvlOpsPath, "skip-top-level-ops", "", "(config file to skip top level operations)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") @@ -148,7 +146,6 @@ func main() { GatewayUrl: gatewayUrl, UseLogsNewSchema: useLogsNewSchema, UseTraceNewSchema: useTraceNewSchema, - UseLicensesV3: useLicensesV3, } // Read the jwt secret key diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 3e25ab23c8..e14eec7ef6 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -113,7 +113,6 @@ type APIHandler struct { UseLogsNewSchema bool UseTraceNewSchema bool - UseLicensesV3 bool hostsRepo *inframetrics.HostsRepo processesRepo *inframetrics.ProcessesRepo @@ -166,8 +165,6 @@ type APIHandlerOpts struct { UseLogsNewSchema bool UseTraceNewSchema bool - // Use Licenses V3 structure - UseLicensesV3 bool } // NewAPIHandler returns an APIHandler @@ -230,7 +227,6 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { querierV2: querierv2, UseLogsNewSchema: opts.UseLogsNewSchema, UseTraceNewSchema: opts.UseTraceNewSchema, - UseLicensesV3: opts.UseLicensesV3, hostsRepo: hostsRepo, processesRepo: processesRepo, podsRepo: podsRepo, From 2701ae5c34f85cb62e0fa2cbc5185fbd4659c13f Mon Sep 17 00:00:00 2001 From: Vikrant Gupta Date: Mon, 16 Dec 2024 22:25:37 +0530 Subject: [PATCH 3/6] fix: unable to remove query tags from the beginning when similar tags present (#6645) * fix: unable to remove query tags from the begining * fix: focus only when the filters dropdown is opened * fix: used auto focus prop --- .../filters/QueryBuilderSearchV2/QueryBuilderSearchV2.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2.tsx b/frontend/src/container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2.tsx index b3d2b2537a..b273de05ae 100644 --- a/frontend/src/container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2.tsx +++ b/frontend/src/container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2.tsx @@ -897,12 +897,14 @@ function QueryBuilderSearchV2(