From 6c5793b32ba8024951461ac658badf7a87db39dd Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Tue, 14 May 2024 11:17:42 +0530 Subject: [PATCH 1/9] feat: initial --- repository/utils.go | 220 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 repository/utils.go diff --git a/repository/utils.go b/repository/utils.go new file mode 100644 index 000000000..f66c98168 --- /dev/null +++ b/repository/utils.go @@ -0,0 +1,220 @@ +package repository + +import ( + "fmt" + "strconv" + "strings" + + "github.com/tailwarden/komiser/models" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func generateFilterQuery(db *bun.DB, filters []models.Filter, queryBuilder func([]string) (string, error)) (string, error) { + whereQueries := make([]string, 0) + for _, filter := range filters { + switch filter.Field { + case "account", "resource", "provider", "name", "region": + query, err := generateStandardFilterQuery(db, filter, false) + if err != nil { + return "", err + } + whereQueries = append(whereQueries, query) + case "cost": + query, err := generateCostFilterQuery(filter) + if err != nil { + return "", err + } + whereQueries = append(whereQueries, query) + case "relation": + query, err := generateRelationFilterQuery(db, filter) + if err != nil { + return "", err + } + whereQueries = append(whereQueries, query) + case "tags:": + query, err := generateStandardFilterQuery(db, filter, true) + if err != nil { + return "", err + } + whereQueries = append(whereQueries, query) + case "tags": + query, err := generateEmptyFilterQuery(db, filter) + if err != nil { + return "", err + } + whereQueries = append(whereQueries, query) + default: + return "", fmt.Errorf("unsupported field: %s", filter.Field) + } + } + return queryBuilder(whereQueries) +} + +func generateEmptyFilterQuery(db *bun.DB, filter models.Filter) (string, error) { + switch filter.Operator { + case "IS_EMPTY": + if db.Dialect().Name() == dialect.SQLite { + return "json_array_length(tags) = 0", nil + } else { + return "jsonb_array_length(tags) = 0", nil + } + case "IS_NOT_EMPTY": + if db.Dialect().Name() == dialect.SQLite { + return "json_array_length(tags) != 0", nil + } else { + return "jsonb_array_length(tags) != 0", nil + } + } + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) +} + +func generateStandardFilterQuery(db *bun.DB, filter models.Filter, withTag bool) (string, error) { + for i := 0; i < len(filter.Values); i++ { + filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) + } + key := strings.ReplaceAll(filter.Field, "tag:", "") + switch filter.Operator { + case "IS": + if withTag { + query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' IN (%s)))", key, strings.Join(filter.Values, ",")) + if db.Dialect().Name() == dialect.SQLite { + query = fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') IN (%s)))", key, strings.Join(filter.Values, ",")) + } + return query, nil + } + return fmt.Sprintf("(%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil + case "IS_NOT": + if withTag { + query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' NOT IN (%s)))", key, strings.Join(filter.Values, ",")) + if db.Dialect().Name() == dialect.SQLite { + query = fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') NOT IN (%s)))", key, strings.Join(filter.Values, ",")) + } + return query, nil + } + return fmt.Sprintf("(%s NOT IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil + case "CONTAINS": + queries := make([]string, 0) + specialChar := "%" + for i := 0; i < len(filter.Values); i++ { + queries = append(queries, fmt.Sprintf("(%s LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) + } + return fmt.Sprintf("(%s)", strings.Join(queries, " OR ")), nil + case "NOT_CONTAINS": + queries := make([]string, 0) + specialChar := "%" + for i := 0; i < len(filter.Values); i++ { + queries = append(queries, fmt.Sprintf("(%s NOT LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) + } + return fmt.Sprintf("(%s)", strings.Join(queries, " AND ")), nil + case "IS_EMPTY": + if withTag { + if db.Dialect().Name() == dialect.SQLite { + return fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') != ''))", key), nil + } else { + return fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' != ''))", key), nil + } + } + return fmt.Sprintf("((coalesce(%s, '') = ''))", filter.Field), nil + case "IS_NOT_EMPTY": + if withTag { + if db.Dialect().Name() == dialect.SQLite { + return fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') != ''))", key), nil + } else { + return fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' != ''))", key), nil + } + } + return fmt.Sprintf("((coalesce(%s, '') != ''))", filter.Field), nil + case "EXISTS": + if db.Dialect().Name() == dialect.SQLite { + return fmt.Sprintf("((json_extract(value, '$.key') = '%s'))", key), nil + } else { + return fmt.Sprintf("((res->>'key' = '%s'))", key), nil + } + case "NOT_EXISTS": + if db.Dialect().Name() == dialect.SQLite { + return fmt.Sprintf(`(NOT EXISTS (SELECT 1 FROM json_each(resources.tags) WHERE (json_extract(value, '$.key') = '%s')))`, key), nil + } else { + return fmt.Sprintf("((res->>'key' != '%s'))", key), nil + } + default: + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) + } +} + +func generateCostFilterQuery(filter models.Filter) (string, error) { + switch filter.Operator { + case "EQUAL": + value, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost = %f)", value), nil + case "BETWEEN": + min, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + max, err := strconv.ParseFloat(filter.Values[1], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost >= %f AND cost <= %f)", min, max), nil + case "GREATER_THAN": + cost, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost > %f)", cost), err + case "LESS_THAN": + cost, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost < %f)", cost), nil + default: + return "", fmt.Errorf("unsupported operator for cost field: %s", filter.Operator) + } +} + +func generateRelationFilterQuery(db *bun.DB, filter models.Filter) (string, error) { + switch filter.Operator { + case "EQUAL": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + if db.Dialect().Name() == dialect.SQLite { + return fmt.Sprintf("json_array_length(resources.relations) = %d", relations), err + } else { + return fmt.Sprintf("jsonb_array_length(resources.relations) = %d", relations), err + } + case "GREATER_THAN": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + if db.Dialect().Name() == dialect.SQLite { + return fmt.Sprintf("json_array_length(resources.relations) > %d", relations), err + } else { + return fmt.Sprintf("jsonb_array_length(resources.relations) > %d", relations), err + } + case "LESS_THAN": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + if db.Dialect().Name() == dialect.SQLite { + return fmt.Sprintf("json_array_length(resources.relations) < %d", relations), err + } else { + return fmt.Sprintf("jsonb_array_length(resources.relations) < %d", relations), err + } + default: + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) + } +} + +func AppendResourceQuery(whereQueries []string) (string, error) { + + return "", nil +} \ No newline at end of file From f9a5f9ed8ad17bbee5b610efd880c48f57de0cb3 Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Tue, 14 May 2024 17:27:18 +0530 Subject: [PATCH 2/9] feat: added dialect support --- controller/controller.go | 2 + repository/core.go | 3 + repository/postgres/postgres.go | 258 ++++++++++++++++++++++++++++++++ repository/sqlite/sqlite.go | 258 ++++++++++++++++++++++++++++++++ repository/utils.go | 220 --------------------------- 5 files changed, 521 insertions(+), 220 deletions(-) delete mode 100644 repository/utils.go diff --git a/controller/controller.go b/controller/controller.go index 55cefc42e..934dcb4fc 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" + "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/repository" ) @@ -33,6 +34,7 @@ type accountOutput struct { type Repository interface { HandleQuery(context.Context, repository.QueryType, interface{}, [][3]string) (sql.Result, error) + GenerateFilterQuery(view models.View, queryTitle string, arguments []int64, queryParameter string) ([]string, error) } type Controller struct { diff --git a/repository/core.go b/repository/core.go index 20d517a4f..bad3dbb67 100644 --- a/repository/core.go +++ b/repository/core.go @@ -44,6 +44,9 @@ const ( ListProvidersKey = "LIST_PROVIDERS" ListServicesKey = "LIST_SERVICES" ListAccountsKey = "LIST_ACCOUNTS" + ListResourceWithFilter = "LIST_RESOURCE_WITH_FILTER" + ListRelationWithFilter = "LIST_RELATION_WITH_FILTER" + ListStatsWithFilter = "LIST_STATS_WITH_FILTER" ) func ExecuteRaw(ctx context.Context, db *bun.DB, query string, schema interface{}, additionals [][3]string) error { diff --git a/repository/postgres/postgres.go b/repository/postgres/postgres.go index b216bcb67..2ecfb6a0c 100644 --- a/repository/postgres/postgres.go +++ b/repository/postgres/postgres.go @@ -3,7 +3,12 @@ package sql import ( "context" "database/sql" + "encoding/json" + "fmt" + "strconv" + "strings" + "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/repository" "github.com/uptrace/bun" ) @@ -90,6 +95,37 @@ var Queries = map[string]repository.Object{ Type: repository.RAW, Query: "SELECT DISTINCT(account) FROM resources", }, + repository.ListResourceWithFilter : { + Type: repository.RAW, + Query: "", + Params: []string{ + "(name LIKE '%%%s%%' OR region LIKE '%%%s%%' OR service LIKE '%%%s%%' OR provider LIKE '%%%s%%' OR account LIKE '%%%s%%' OR (value->>'key' LIKE '%%%s%%') OR (value->>'value' LIKE '%%%s%%'))", + "SELECT * FROM resources CROSS JOIN jsonb_array_elements(tags) WHERE %s ORDER BY id LIMIT %d OFFSET %d", + "SELECT * FROM resources ORDER BY id LIMIT %d OFFSET %d", + "SELECT DISTINCT id, resource_id, provider, account, service, region, name, created_at, fetched_at,cost, metadata, tags,link FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s ", + "SELECT * FROM resources WHERE %s ORDER BY id LIMIT %d OFFSET %d", + "SELECT * FROM resources WHERE %s AND id NOT IN (%s) ORDER BY id LIMIT %d OFFSET %d", + }, + }, + repository.ListRelationWithFilter : { + Type: repository.RAW, + Query: "", + Params: []string{ + "SELECT DISTINCT resources.resource_id, resources.provider, resources.name, resources.service, resources.relations FROM resources WHERE (jsonb_array_length(relations) > 0)", + }, + }, + repository.ListStatsWithFilter : { + Type: repository.RAW, + Query: "", + Params: []string{ + "SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s) AS temp", + "SELECT COUNT(*) as count FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s", + "SELECT SUM(cost) as sum FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s", + "SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources WHERE %s) AS temp", + "SELECT COUNT(*) as count FROM resources WHERE %s", + "SELECT SUM(cost) as sum FROM resources WHERE %s", + }, + }, } func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, schema interface{}, conditions [][3]string) (sql.Result, error) { @@ -117,3 +153,225 @@ func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, sche } return resp, err } + +func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, arguments []int64, queryParameter string) ([]string, error) { + whereQueries := make([]string, 0) + filterWithTags := false + for _, filter := range view.Filters { + switch filter.Field { + case "account", "resource", "provider", "name", "region": + query, err := generateStandardFilterQuery(filter, false) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + case "cost": + query, err := generateCostFilterQuery(filter) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + case "relation": + query, err := generateRelationFilterQuery(filter) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + case "tags:": + filterWithTags = true + query, err := generateStandardFilterQuery(filter, true) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + case "tags": + query, err := generateEmptyFilterQuery(filter) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + default: + return nil, fmt.Errorf("unsupported field: %s", filter.Field) + } + } + + whereClause := strings.Join(whereQueries, " AND ") + return queryBuilderWithFilter(view, queryTitle, arguments, queryParameter, filterWithTags, whereClause), nil +} + +func queryBuilderWithFilter(view models.View, queryTitle string, arguments []int64, query string, withTags bool, whereClause string) []string { + searchQuery := []string{} + limit, skip := arguments[0], arguments[1] + if len(view.Filters) == 0 { + switch queryTitle { + case repository.ListRelationWithFilter: + return append(searchQuery, Queries[queryTitle].Params[0]) + case repository.ListResourceWithFilter: + tempQuery := "" + if len(query) > 0 { + whereClause = fmt.Sprintf(Queries[queryTitle].Params[0], query, query, query, query, query, query, query) + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[1], whereClause, limit, skip) + } else { + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[2], limit, skip) + } + return append(searchQuery, tempQuery) + } + } else if queryTitle == repository.ListRelationWithFilter { + return append(searchQuery, Queries[queryTitle].Params[0] + " AND " + whereClause) + } + + if withTags { + if queryTitle == repository.ListStatsWithFilter { + for i := 0; i<3; i++ { + searchQuery = append(searchQuery, fmt.Sprintf(Queries[queryTitle].Params[i], whereClause)) + } + return searchQuery + } + tempQuery := fmt.Sprintf(Queries[queryTitle].Params[3]+"ORDER BY id LIMIT %d OFFSET %d", whereClause, limit, skip) + if len(view.Exclude) > 0 { + s, _ := json.Marshal(view.Exclude) + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[3]+"AND id NOT IN (%s) ORDER BY id LIMIT %d OFFSET %d", whereClause, strings.Trim(string(s), "[]"), limit, skip) + } + return append(searchQuery, tempQuery) + } else { + if queryTitle == repository.ListStatsWithFilter { + for i := 3; i<6; i++ { + searchQuery = append(searchQuery, fmt.Sprintf(Queries[queryTitle].Params[i], whereClause)) + } + return searchQuery + } + tempQuery := fmt.Sprintf(Queries[queryTitle].Params[4], whereClause, limit, skip) + + if whereClause == "" { + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[2], limit, skip) + } + + if len(view.Exclude) > 0 { + s, _ := json.Marshal(view.Exclude) + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[5], whereClause, strings.Trim(string(s), "[]"), limit, skip) + } + + return append(searchQuery, tempQuery) + } +} + +func generateEmptyFilterQuery(filter models.Filter) (string, error) { + switch filter.Operator { + case "IS_EMPTY": + return "jsonb_array_length(tags) = 0", nil + case "IS_NOT_EMPTY": + return "jsonb_array_length(tags) != 0", nil + } + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) +} + +func generateStandardFilterQuery(filter models.Filter, withTag bool) (string, error) { + for i := 0; i < len(filter.Values); i++ { + filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) + } + key := strings.ReplaceAll(filter.Field, "tag:", "") + switch filter.Operator { + case "IS": + if withTag { + query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' IN (%s)))", key, strings.Join(filter.Values, ",")) + return query, nil + } + return fmt.Sprintf("(%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil + case "IS_NOT": + if withTag { + query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' NOT IN (%s)))", key, strings.Join(filter.Values, ",")) + return query, nil + } + return fmt.Sprintf("(%s NOT IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil + case "CONTAINS": + queries := make([]string, 0) + specialChar := "%" + for i := 0; i < len(filter.Values); i++ { + queries = append(queries, fmt.Sprintf("(%s LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) + } + return fmt.Sprintf("(%s)", strings.Join(queries, " OR ")), nil + case "NOT_CONTAINS": + queries := make([]string, 0) + specialChar := "%" + for i := 0; i < len(filter.Values); i++ { + queries = append(queries, fmt.Sprintf("(%s NOT LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) + } + return fmt.Sprintf("(%s)", strings.Join(queries, " AND ")), nil + case "IS_EMPTY": + if withTag { + return fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' != ''))", key), nil + } + return fmt.Sprintf("((coalesce(%s, '') = ''))", filter.Field), nil + case "IS_NOT_EMPTY": + if withTag { + return fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' != ''))", key), nil + } + return fmt.Sprintf("((coalesce(%s, '') != ''))", filter.Field), nil + case "EXISTS": + return fmt.Sprintf("((res->>'key' = '%s'))", key), nil + case "NOT_EXISTS": + return fmt.Sprintf("((res->>'key' != '%s'))", key), nil + default: + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) + } +} + +func generateCostFilterQuery(filter models.Filter) (string, error) { + switch filter.Operator { + case "EQUAL": + value, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost = %f)", value), nil + case "BETWEEN": + min, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + max, err := strconv.ParseFloat(filter.Values[1], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost >= %f AND cost <= %f)", min, max), nil + case "GREATER_THAN": + cost, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost > %f)", cost), err + case "LESS_THAN": + cost, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost < %f)", cost), nil + default: + return "", fmt.Errorf("unsupported operator for cost field: %s", filter.Operator) + } +} + +func generateRelationFilterQuery(filter models.Filter) (string, error) { + switch filter.Operator { + case "EQUAL": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + return fmt.Sprintf("jsonb_array_length(resources.relations) = %d", relations), err + case "GREATER_THAN": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + return fmt.Sprintf("jsonb_array_length(resources.relations) > %d", relations), err + case "LESS_THAN": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + return fmt.Sprintf("jsonb_array_length(resources.relations) < %d", relations), err + default: + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) + } +} \ No newline at end of file diff --git a/repository/sqlite/sqlite.go b/repository/sqlite/sqlite.go index cf329c43c..75d2b1c2e 100644 --- a/repository/sqlite/sqlite.go +++ b/repository/sqlite/sqlite.go @@ -3,7 +3,12 @@ package sql import ( "context" "database/sql" + "encoding/json" + "fmt" + "strconv" + "strings" + "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/repository" "github.com/uptrace/bun" ) @@ -91,6 +96,37 @@ var Queries = map[string]repository.Object{ Type: repository.RAW, Query: "SELECT DISTINCT(account) FROM resources", }, + repository.ListResourceWithFilter : { + Type: repository.RAW, + Query: "", + Params: []string{ + "(name LIKE '%%%s%%' OR region LIKE '%%%s%%' OR service LIKE '%%%s%%' OR provider LIKE '%%%s%%' OR account LIKE '%%%s%%' OR (tags LIKE '%%%s%%'))", + "SELECT * FROM resources WHERE %s ORDER BY id LIMIT %d OFFSET %d", + "SELECT * FROM resources ORDER BY id LIMIT %d OFFSET %d", + "SELECT DISTINCT resources.id, resources.resource_id, resources.provider, resources.account, resources.service, resources.region, resources.name, resources.created_at, resources.fetched_at, resources.cost, resources.metadata, resources.tags, resources.link FROM resources CROSS JOIN json_each(tags) WHERE ", + "SELECT * FROM resources WHERE %s ORDER BY id LIMIT %d OFFSET %d", + "SELECT * FROM resources WHERE %s AND id NOT IN (%s) ORDER BY id LIMIT %d OFFSET %d", + }, + }, + repository.ListRelationWithFilter : { + Type: repository.RAW, + Query: "", + Params: []string{ + "SELECT DISTINCT resources.resource_id, resources.provider, resources.name, resources.service, resources.relations FROM resources WHERE (json_array_length(relations) > 0)", + }, + }, + repository.ListStatsWithFilter : { + Type: repository.RAW, + Query: "", + Params: []string{ + "SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s) AS temp", + "SELECT COUNT(*) as count FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s", + "SELECT SUM(cost) as sum FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s", + "SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources WHERE %s) AS temp", + "SELECT COUNT(*) as count FROM resources WHERE %s", + "SELECT SUM(cost) as sum FROM resources WHERE %s", + }, + }, } func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, schema interface{}, conditions [][3]string) (sql.Result, error) { @@ -118,3 +154,225 @@ func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, sche } return resp, err } + +func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, arguments []int64, queryParameter string) ([]string, error) { + whereQueries := make([]string, 0) + filterWithTags := false + for _, filter := range view.Filters { + switch filter.Field { + case "account", "resource", "provider", "name", "region": + query, err := generateStandardFilterQuery(filter, false) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + case "cost": + query, err := generateCostFilterQuery(filter) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + case "relation": + query, err := generateRelationFilterQuery(filter) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + case "tags:": + filterWithTags = true + query, err := generateStandardFilterQuery(filter, true) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + case "tags": + query, err := generateEmptyFilterQuery(filter) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + default: + return nil, fmt.Errorf("unsupported field: %s", filter.Field) + } + } + + whereClause := strings.Join(whereQueries, " AND ") + return queryBuilderWithFilter(view, queryTitle, arguments, queryParameter, filterWithTags, whereClause), nil +} + +func queryBuilderWithFilter(view models.View, queryTitle string, arguments []int64, query string, withTags bool, whereClause string) []string { + searchQuery := []string{} + limit, skip := arguments[0], arguments[1] + if len(view.Filters) == 0 { + switch queryTitle { + case repository.ListRelationWithFilter: + return append(searchQuery, Queries[queryTitle].Params[0]) + case repository.ListResourceWithFilter: + tempQuery := "" + if len(query) > 0 { + whereClause = fmt.Sprintf(Queries[queryTitle].Params[0], query, query, query, query, query, query, query) + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[1], whereClause, limit, skip) + } else { + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[2], limit, skip) + } + return append(searchQuery, tempQuery) + } + } else if queryTitle == repository.ListRelationWithFilter { + return append(searchQuery, Queries[queryTitle].Params[0] + " AND " + whereClause) + } + + if withTags { + if queryTitle == repository.ListStatsWithFilter { + for i := 0; i<3; i++ { + searchQuery = append(searchQuery, fmt.Sprintf(Queries[queryTitle].Params[i], whereClause)) + } + return searchQuery + } + tempQuery := fmt.Sprintf(Queries[queryTitle].Params[3]+"type='object' AND %s ORDER BY resources.id LIMIT %d OFFSET %d", whereClause, limit, skip) + if len(view.Exclude) > 0 { + s, _ := json.Marshal(view.Exclude) + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[3]+"resources.id NOT IN (%s) AND type='object' AND %s ORDER BY resources.id LIMIT %d OFFSET %d", whereClause, strings.Trim(string(s), "[]"), limit, skip) + } + return append(searchQuery, tempQuery) + } else { + if queryTitle == repository.ListStatsWithFilter { + for i := 3; i<6; i++ { + searchQuery = append(searchQuery, fmt.Sprintf(Queries[queryTitle].Params[i], whereClause)) + } + return searchQuery + } + tempQuery := fmt.Sprintf(Queries[queryTitle].Params[4], whereClause, limit, skip) + + if whereClause == "" { + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[2], limit, skip) + } + + if len(view.Exclude) > 0 { + s, _ := json.Marshal(view.Exclude) + tempQuery = fmt.Sprintf(Queries[queryTitle].Params[5], whereClause, strings.Trim(string(s), "[]"), limit, skip) + } + + return append(searchQuery, tempQuery) + } +} + +func generateEmptyFilterQuery(filter models.Filter) (string, error) { + switch filter.Operator { + case "IS_EMPTY": + return "json_array_length(tags) = 0", nil + case "IS_NOT_EMPTY": + return "json_array_length(tags) != 0", nil + } + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) +} + +func generateStandardFilterQuery(filter models.Filter, withTag bool) (string, error) { + for i := 0; i < len(filter.Values); i++ { + filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) + } + key := strings.ReplaceAll(filter.Field, "tag:", "") + switch filter.Operator { + case "IS": + if withTag { + query := fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') IN (%s)))", key, strings.Join(filter.Values, ",")) + return query, nil + } + return fmt.Sprintf("(%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil + case "IS_NOT": + if withTag { + query := fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') NOT IN (%s)))", key, strings.Join(filter.Values, ",")) + return query, nil + } + return fmt.Sprintf("(%s NOT IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil + case "CONTAINS": + queries := make([]string, 0) + specialChar := "%" + for i := 0; i < len(filter.Values); i++ { + queries = append(queries, fmt.Sprintf("(%s LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) + } + return fmt.Sprintf("(%s)", strings.Join(queries, " OR ")), nil + case "NOT_CONTAINS": + queries := make([]string, 0) + specialChar := "%" + for i := 0; i < len(filter.Values); i++ { + queries = append(queries, fmt.Sprintf("(%s NOT LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) + } + return fmt.Sprintf("(%s)", strings.Join(queries, " AND ")), nil + case "IS_EMPTY": + if withTag { + return fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') != ''))", key), nil + } + return fmt.Sprintf("((coalesce(%s, '') = ''))", filter.Field), nil + case "IS_NOT_EMPTY": + if withTag { + return fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') != ''))", key), nil + } + return fmt.Sprintf("((coalesce(%s, '') != ''))", filter.Field), nil + case "EXISTS": + return fmt.Sprintf("((json_extract(value, '$.key') = '%s'))", key), nil + case "NOT_EXISTS": + return fmt.Sprintf(`(NOT EXISTS (SELECT 1 FROM json_each(resources.tags) WHERE (json_extract(value, '$.key') = '%s')))`, key), nil + default: + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) + } +} + +func generateCostFilterQuery(filter models.Filter) (string, error) { + switch filter.Operator { + case "EQUAL": + value, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost = %f)", value), nil + case "BETWEEN": + min, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + max, err := strconv.ParseFloat(filter.Values[1], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost >= %f AND cost <= %f)", min, max), nil + case "GREATER_THAN": + cost, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost > %f)", cost), err + case "LESS_THAN": + cost, err := strconv.ParseFloat(filter.Values[0], 64) + if err != nil { + return "", err + } + return fmt.Sprintf("(cost < %f)", cost), nil + default: + return "", fmt.Errorf("unsupported operator for cost field: %s", filter.Operator) + } +} + +func generateRelationFilterQuery(filter models.Filter) (string, error) { + switch filter.Operator { + case "EQUAL": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + return fmt.Sprintf("json_array_length(resources.relations) = %d", relations), err + case "GREATER_THAN": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + return fmt.Sprintf("json_array_length(resources.relations) > %d", relations), err + case "LESS_THAN": + relations, err := strconv.Atoi(filter.Values[0]) + if err != nil { + return "", err + } + return fmt.Sprintf("json_array_length(resources.relations) < %d", relations), err + default: + return "", fmt.Errorf("unsupported operator: %s", filter.Operator) + } +} \ No newline at end of file diff --git a/repository/utils.go b/repository/utils.go deleted file mode 100644 index f66c98168..000000000 --- a/repository/utils.go +++ /dev/null @@ -1,220 +0,0 @@ -package repository - -import ( - "fmt" - "strconv" - "strings" - - "github.com/tailwarden/komiser/models" - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func generateFilterQuery(db *bun.DB, filters []models.Filter, queryBuilder func([]string) (string, error)) (string, error) { - whereQueries := make([]string, 0) - for _, filter := range filters { - switch filter.Field { - case "account", "resource", "provider", "name", "region": - query, err := generateStandardFilterQuery(db, filter, false) - if err != nil { - return "", err - } - whereQueries = append(whereQueries, query) - case "cost": - query, err := generateCostFilterQuery(filter) - if err != nil { - return "", err - } - whereQueries = append(whereQueries, query) - case "relation": - query, err := generateRelationFilterQuery(db, filter) - if err != nil { - return "", err - } - whereQueries = append(whereQueries, query) - case "tags:": - query, err := generateStandardFilterQuery(db, filter, true) - if err != nil { - return "", err - } - whereQueries = append(whereQueries, query) - case "tags": - query, err := generateEmptyFilterQuery(db, filter) - if err != nil { - return "", err - } - whereQueries = append(whereQueries, query) - default: - return "", fmt.Errorf("unsupported field: %s", filter.Field) - } - } - return queryBuilder(whereQueries) -} - -func generateEmptyFilterQuery(db *bun.DB, filter models.Filter) (string, error) { - switch filter.Operator { - case "IS_EMPTY": - if db.Dialect().Name() == dialect.SQLite { - return "json_array_length(tags) = 0", nil - } else { - return "jsonb_array_length(tags) = 0", nil - } - case "IS_NOT_EMPTY": - if db.Dialect().Name() == dialect.SQLite { - return "json_array_length(tags) != 0", nil - } else { - return "jsonb_array_length(tags) != 0", nil - } - } - return "", fmt.Errorf("unsupported operator: %s", filter.Operator) -} - -func generateStandardFilterQuery(db *bun.DB, filter models.Filter, withTag bool) (string, error) { - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - key := strings.ReplaceAll(filter.Field, "tag:", "") - switch filter.Operator { - case "IS": - if withTag { - query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' IN (%s)))", key, strings.Join(filter.Values, ",")) - if db.Dialect().Name() == dialect.SQLite { - query = fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') IN (%s)))", key, strings.Join(filter.Values, ",")) - } - return query, nil - } - return fmt.Sprintf("(%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil - case "IS_NOT": - if withTag { - query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' NOT IN (%s)))", key, strings.Join(filter.Values, ",")) - if db.Dialect().Name() == dialect.SQLite { - query = fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') NOT IN (%s)))", key, strings.Join(filter.Values, ",")) - } - return query, nil - } - return fmt.Sprintf("(%s NOT IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil - case "CONTAINS": - queries := make([]string, 0) - specialChar := "%" - for i := 0; i < len(filter.Values); i++ { - queries = append(queries, fmt.Sprintf("(%s LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } - return fmt.Sprintf("(%s)", strings.Join(queries, " OR ")), nil - case "NOT_CONTAINS": - queries := make([]string, 0) - specialChar := "%" - for i := 0; i < len(filter.Values); i++ { - queries = append(queries, fmt.Sprintf("(%s NOT LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } - return fmt.Sprintf("(%s)", strings.Join(queries, " AND ")), nil - case "IS_EMPTY": - if withTag { - if db.Dialect().Name() == dialect.SQLite { - return fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') != ''))", key), nil - } else { - return fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' != ''))", key), nil - } - } - return fmt.Sprintf("((coalesce(%s, '') = ''))", filter.Field), nil - case "IS_NOT_EMPTY": - if withTag { - if db.Dialect().Name() == dialect.SQLite { - return fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') != ''))", key), nil - } else { - return fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' != ''))", key), nil - } - } - return fmt.Sprintf("((coalesce(%s, '') != ''))", filter.Field), nil - case "EXISTS": - if db.Dialect().Name() == dialect.SQLite { - return fmt.Sprintf("((json_extract(value, '$.key') = '%s'))", key), nil - } else { - return fmt.Sprintf("((res->>'key' = '%s'))", key), nil - } - case "NOT_EXISTS": - if db.Dialect().Name() == dialect.SQLite { - return fmt.Sprintf(`(NOT EXISTS (SELECT 1 FROM json_each(resources.tags) WHERE (json_extract(value, '$.key') = '%s')))`, key), nil - } else { - return fmt.Sprintf("((res->>'key' != '%s'))", key), nil - } - default: - return "", fmt.Errorf("unsupported operator: %s", filter.Operator) - } -} - -func generateCostFilterQuery(filter models.Filter) (string, error) { - switch filter.Operator { - case "EQUAL": - value, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - return "", err - } - return fmt.Sprintf("(cost = %f)", value), nil - case "BETWEEN": - min, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - return "", err - } - max, err := strconv.ParseFloat(filter.Values[1], 64) - if err != nil { - return "", err - } - return fmt.Sprintf("(cost >= %f AND cost <= %f)", min, max), nil - case "GREATER_THAN": - cost, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - return "", err - } - return fmt.Sprintf("(cost > %f)", cost), err - case "LESS_THAN": - cost, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - return "", err - } - return fmt.Sprintf("(cost < %f)", cost), nil - default: - return "", fmt.Errorf("unsupported operator for cost field: %s", filter.Operator) - } -} - -func generateRelationFilterQuery(db *bun.DB, filter models.Filter) (string, error) { - switch filter.Operator { - case "EQUAL": - relations, err := strconv.Atoi(filter.Values[0]) - if err != nil { - return "", err - } - if db.Dialect().Name() == dialect.SQLite { - return fmt.Sprintf("json_array_length(resources.relations) = %d", relations), err - } else { - return fmt.Sprintf("jsonb_array_length(resources.relations) = %d", relations), err - } - case "GREATER_THAN": - relations, err := strconv.Atoi(filter.Values[0]) - if err != nil { - return "", err - } - if db.Dialect().Name() == dialect.SQLite { - return fmt.Sprintf("json_array_length(resources.relations) > %d", relations), err - } else { - return fmt.Sprintf("jsonb_array_length(resources.relations) > %d", relations), err - } - case "LESS_THAN": - relations, err := strconv.Atoi(filter.Values[0]) - if err != nil { - return "", err - } - if db.Dialect().Name() == dialect.SQLite { - return fmt.Sprintf("json_array_length(resources.relations) < %d", relations), err - } else { - return fmt.Sprintf("jsonb_array_length(resources.relations) < %d", relations), err - } - default: - return "", fmt.Errorf("unsupported operator: %s", filter.Operator) - } -} - -func AppendResourceQuery(whereQueries []string) (string, error) { - - return "", nil -} \ No newline at end of file From 1ba9d64f5c6b0dd06b3929dbed9b01aaadb890dd Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Sat, 18 May 2024 19:45:47 +0530 Subject: [PATCH 3/9] feat: added resource with filters to controller --- controller/controller.go | 2 +- controller/resources.go | 20 +++ handlers/resources_handler.go | 230 +------------------------------- repository/postgres/postgres.go | 40 ++++-- repository/sqlite/sqlite.go | 43 ++++-- 5 files changed, 86 insertions(+), 249 deletions(-) diff --git a/controller/controller.go b/controller/controller.go index 259ebc357..7f1713615 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -5,7 +5,6 @@ import ( "database/sql" "github.com/tailwarden/komiser/models" - "github.com/tailwarden/komiser/repository" ) type totalOutput struct { @@ -35,6 +34,7 @@ type accountOutput struct { type Repository interface { HandleQuery(context.Context, string, interface{}, [][3]string) (sql.Result, error) GenerateFilterQuery(view models.View, queryTitle string, arguments []int64, queryParameter string) ([]string, error) + UpdateQuery(query, queryTitle string) error } type Controller struct { diff --git a/controller/resources.go b/controller/resources.go index 5bcc74598..d4902248c 100644 --- a/controller/resources.go +++ b/controller/resources.go @@ -14,11 +14,13 @@ func (ctrl *Controller) GetResource(c context.Context, resourceId string) (resou } func (ctrl *Controller) GetResources(c context.Context, idList string) (resources []models.Resource, err error) { + resources = make([]models.Resource, 0) _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &resources, [][3]string{{"id", "IN", "(" + strings.Trim(idList, "[]") + ")"}}) return } func (ctrl *Controller) ListResources(c context.Context) (resources []models.Resource, err error) { + resources = make([]models.Resource, 0) _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &resources, [][3]string{}) return } @@ -37,3 +39,21 @@ func (ctrl *Controller) SumResourceCost(c context.Context) (cost costOutput, err _, err = ctrl.repo.HandleQuery(c, repository.ResourceCostSumKey, &cost, [][3]string{}) return } + +func (ctrl *Controller) ResourceWithFilter(c context.Context, view models.View, arguments []int64, queryParameter string) (resources []models.Resource, err error) { + resources = make([]models.Resource, 0) + queries, err := ctrl.repo.GenerateFilterQuery(view, repository.ListResourceWithFilter, arguments, queryParameter) + if err != nil { + return + } + for _, query := range queries { + if err = ctrl.repo.UpdateQuery(query, repository.ListResourceWithFilter); err != nil { + return + } + _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}) + if err != nil { + return + } + } + return +} \ No newline at end of file diff --git a/handlers/resources_handler.go b/handlers/resources_handler.go index 84f4f5ae3..ea5badd74 100644 --- a/handlers/resources_handler.go +++ b/handlers/resources_handler.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" "github.com/tailwarden/komiser/controller" "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/repository/postgres" @@ -53,11 +52,13 @@ func NewApiHandler(ctx context.Context, telemetry bool, analytics utils.Analytic func (handler *ApiHandler) FilterResourcesHandler(c *gin.Context) { var filters []models.Filter + resources := make([]models.Resource, 0) limitRaw := c.Query("limit") skipRaw := c.Query("skip") query := c.Query("query") viewId := c.Query("view") + queryParameter := query view := new(models.View) if viewId != "" { @@ -89,229 +90,12 @@ func (handler *ApiHandler) FilterResourcesHandler(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - - filterWithTags := false - whereQueries := make([]string, 0) - for _, filter := range filters { - if filter.Field == "name" || filter.Field == "region" || filter.Field == "service" || filter.Field == "provider" || filter.Field == "account" { - switch filter.Operator { - case "IS": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("(%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")) - whereQueries = append(whereQueries, query) - case "IS_NOT": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("(%s NOT IN (%s))", filter.Field, strings.Join(filter.Values, ",")) - whereQueries = append(whereQueries, query) - case "CONTAINS": - queries := make([]string, 0) - specialChar := "%" - for i := 0; i < len(filter.Values); i++ { - queries = append(queries, fmt.Sprintf("(%s LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } - whereQueries = append(whereQueries, fmt.Sprintf("(%s)", strings.Join(queries, " OR "))) - case "NOT_CONTAINS": - queries := make([]string, 0) - specialChar := "%" - for i := 0; i < len(filter.Values); i++ { - queries = append(queries, fmt.Sprintf("(%s NOT LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } - whereQueries = append(whereQueries, fmt.Sprintf("(%s)", strings.Join(queries, " AND "))) - case "IS_EMPTY": - whereQueries = append(whereQueries, fmt.Sprintf("((coalesce(%s, '') = ''))", filter.Field)) - case "IS_NOT_EMPTY": - whereQueries = append(whereQueries, fmt.Sprintf("((coalesce(%s, '') != ''))", filter.Field)) - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } else if strings.HasPrefix(filter.Field, "tag:") { - filterWithTags = true - key := strings.ReplaceAll(filter.Field, "tag:", "") - switch filter.Operator { - case "CONTAINS": - case "IS": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' IN (%s)))", key, strings.Join(filter.Values, ",")) - if handler.db.Dialect().Name() == dialect.SQLite { - query = fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') IN (%s)))", key, strings.Join(filter.Values, ",")) - } - whereQueries = append(whereQueries, query) - case "NOT_CONTAINS": - case "IS_NOT": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' NOT IN (%s)))", key, strings.Join(filter.Values, ",")) - if handler.db.Dialect().Name() == dialect.SQLite { - query = fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') NOT IN (%s)))", key, strings.Join(filter.Values, ",")) - } - whereQueries = append(whereQueries, query) - case "IS_EMPTY": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') = ''))", key)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' = ''))", key)) - } - case "IS_NOT_EMPTY": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') != ''))", key)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' != ''))", key)) - } - case "EXISTS": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("((json_extract(value, '$.key') = '%s'))", key)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("((res->>'key' = '%s'))", key)) - } - case "NOT_EXISTS": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf(`(NOT EXISTS (SELECT 1 FROM json_each(resources.tags) WHERE (json_extract(value, '$.key') = '%s')))`, key)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("((res->>'key' != '%s'))", key)) - } - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } else if filter.Field == "tags" { - switch filter.Operator { - case "IS_EMPTY": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, "json_array_length(tags) = 0") - } else { - whereQueries = append(whereQueries, "jsonb_array_length(tags) = 0") - } - case "IS_NOT_EMPTY": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, "json_array_length(tags) != 0") - } else { - whereQueries = append(whereQueries, "jsonb_array_length(tags) != 0") - } - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } else if filter.Field == "cost" { - switch filter.Operator { - case "EQUAL": - cost, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - whereQueries = append(whereQueries, fmt.Sprintf("(cost = %f)", cost)) - case "BETWEEN": - min, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - max, err := strconv.ParseFloat(filter.Values[1], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - whereQueries = append(whereQueries, fmt.Sprintf("(cost >= %f AND cost <= %f)", min, max)) - case "GREATER_THAN": - cost, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - whereQueries = append(whereQueries, fmt.Sprintf("(cost > %f)", cost)) - case "LESS_THAN": - cost, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - whereQueries = append(whereQueries, fmt.Sprintf("(cost < %f)", cost)) - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - } else { - c.JSON(http.StatusInternalServerError, gin.H{"error": "field is invalid or not supported"}) - return - } - } - - if len(query) > 0 { - clause := fmt.Sprintf("(name LIKE '%%%s%%' OR region LIKE '%%%s%%' OR service LIKE '%%%s%%' OR provider LIKE '%%%s%%' OR account LIKE '%%%s%%' OR (tags LIKE '%%%s%%'))", query, query, query, query, query, query) - whereQueries = append(whereQueries, clause) - } - - whereClause := strings.Join(whereQueries, " AND ") - - resources := make([]models.Resource, 0) - - if len(filters) == 0 { - if len(query) > 0 { - whereClause := fmt.Sprintf("(name LIKE '%%%s%%' OR region LIKE '%%%s%%' OR service LIKE '%%%s%%' OR provider LIKE '%%%s%%' OR account LIKE '%%%s%%' OR (tags LIKE '%%%s%%'))", query, query, query, query, query, query) - searchQuery := fmt.Sprintf("SELECT * FROM resources WHERE %s ORDER BY id LIMIT %d OFFSET %d", whereClause, limit, skip) - - if handler.db.Dialect().Name() == dialect.PG { - whereClause = fmt.Sprintf("(name LIKE '%%%s%%' OR region LIKE '%%%s%%' OR service LIKE '%%%s%%' OR provider LIKE '%%%s%%' OR account LIKE '%%%s%%' OR (value->>'key' LIKE '%%%s%%') OR (value->>'value' LIKE '%%%s%%'))", query, query, query, query, query, query, query) - searchQuery = fmt.Sprintf("SELECT * FROM resources CROSS JOIN jsonb_array_elements(tags) WHERE %s ORDER BY id LIMIT %d OFFSET %d", whereClause, limit, skip) - } - - err = handler.db.NewRaw(searchQuery).Scan(handler.ctx, &resources) - if err != nil { - logrus.WithError(err).Error("scan failed") - } - } else { - err = handler.db.NewRaw(fmt.Sprintf("SELECT * FROM resources ORDER BY id LIMIT %d OFFSET %d", limit, skip)).Scan(handler.ctx, &resources) - if err != nil { - logrus.WithError(err).Error("scan failed") - } - } - c.JSON(http.StatusOK, resources) + view.Filters = filters + resources, err = handler.ctrl.ResourceWithFilter(c, *view, []int64{limit, skip}, queryParameter) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return - } - - if filterWithTags { - query := fmt.Sprintf("SELECT DISTINCT id, resource_id, provider, account, service, region, name, created_at, fetched_at,cost, metadata, tags,link FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s ORDER BY id LIMIT %d OFFSET %d", whereClause, limit, skip) - if len(view.Exclude) > 0 { - s, _ := json.Marshal(view.Exclude) - query = fmt.Sprintf("SELECT DISTINCT id, resource_id, provider, account, service, region, name, created_at, fetched_at,cost, metadata, tags,link FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s AND id NOT IN (%s) ORDER BY id LIMIT %d OFFSET %d", whereClause, strings.Trim(string(s), "[]"), limit, skip) - } - if handler.db.Dialect().Name() == dialect.SQLite { - query = fmt.Sprintf("SELECT DISTINCT resources.id, resources.resource_id, resources.provider, resources.account, resources.service, resources.region, resources.name, resources.created_at, resources.fetched_at, resources.cost, resources.metadata, resources.tags, resources.link FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s ORDER BY resources.id LIMIT %d OFFSET %d", whereClause, limit, skip) - if len(view.Exclude) > 0 { - s, _ := json.Marshal(view.Exclude) - query = fmt.Sprintf("SELECT DISTINCT resources.id, resources.resource_id, resources.provider, resources.account, resources.service, resources.region, resources.name, resources.created_at, resources.fetched_at, resources.cost, resources.metadata, resources.tags, resources.link FROM resources CROSS JOIN json_each(tags) WHERE resources.id NOT IN (%s) AND type='object' AND %s ORDER BY resources.id LIMIT %d OFFSET %d", strings.Trim(string(s), "[]"), whereClause, limit, skip) - } - } - err = handler.db.NewRaw(query).Scan(handler.ctx, &resources) - if err != nil { - logrus.WithError(err).Error("scan failed") - } - } else { - query := fmt.Sprintf("SELECT * FROM resources WHERE %s ORDER BY id LIMIT %d OFFSET %d", whereClause, limit, skip) - - if whereClause == "" { - query = fmt.Sprintf("SELECT * FROM resources ORDER BY id LIMIT %d OFFSET %d", limit, skip) - } - - if len(view.Exclude) > 0 { - s, _ := json.Marshal(view.Exclude) - query = fmt.Sprintf("SELECT * FROM resources WHERE %s AND id NOT IN (%s) ORDER BY id LIMIT %d OFFSET %d", whereClause, strings.Trim(string(s), "[]"), limit, skip) - } - - err = handler.db.NewRaw(query).Scan(handler.ctx, &resources) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - } + } c.JSON(http.StatusOK, resources) } diff --git a/repository/postgres/postgres.go b/repository/postgres/postgres.go index 0cc54a95d..5d623363e 100644 --- a/repository/postgres/postgres.go +++ b/repository/postgres/postgres.go @@ -160,7 +160,7 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, filterWithTags := false for _, filter := range view.Filters { switch filter.Field { - case "account", "resource", "provider", "name", "region": + case "account", "resource", "service", "provider", "name", "region": query, err := generateStandardFilterQuery(filter, false) if err != nil { return nil, err @@ -178,13 +178,6 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, return nil, err } whereQueries = append(whereQueries, query) - case "tags:": - filterWithTags = true - query, err := generateStandardFilterQuery(filter, true) - if err != nil { - return nil, err - } - whereQueries = append(whereQueries, query) case "tags": query, err := generateEmptyFilterQuery(filter) if err != nil { @@ -192,7 +185,16 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, } whereQueries = append(whereQueries, query) default: - return nil, fmt.Errorf("unsupported field: %s", filter.Field) + if strings.HasPrefix(filter.Field, "tag:") { + filterWithTags = true + query, err := generateStandardFilterQuery(filter, true) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + } else { + return nil, fmt.Errorf("unsupported field: %s", filter.Field) + } } } @@ -200,6 +202,17 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, return queryBuilderWithFilter(view, queryTitle, arguments, queryParameter, filterWithTags, whereClause), nil } +func (repo *Repository) UpdateQuery(query, queryTitle string) error { + obj, exists := repo.queries[queryTitle] + if !exists { + return fmt.Errorf("queryTitle %s not found in repository", queryTitle) + } + obj.Query = query + repo.queries[queryTitle] = obj + + return nil +} + func queryBuilderWithFilter(view models.View, queryTitle string, arguments []int64, query string, withTags bool, whereClause string) []string { searchQuery := []string{} limit, skip := arguments[0], arguments[1] @@ -267,18 +280,21 @@ func generateEmptyFilterQuery(filter models.Filter) (string, error) { } func generateStandardFilterQuery(filter models.Filter, withTag bool) (string, error) { - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } key := strings.ReplaceAll(filter.Field, "tag:", "") switch filter.Operator { case "IS": + for i := 0; i < len(filter.Values); i++ { + filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) + } if withTag { query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' IN (%s)))", key, strings.Join(filter.Values, ",")) return query, nil } return fmt.Sprintf("(%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil case "IS_NOT": + for i := 0; i < len(filter.Values); i++ { + filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) + } if withTag { query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' NOT IN (%s)))", key, strings.Join(filter.Values, ",")) return query, nil diff --git a/repository/sqlite/sqlite.go b/repository/sqlite/sqlite.go index f17d302ba..da8900f79 100644 --- a/repository/sqlite/sqlite.go +++ b/repository/sqlite/sqlite.go @@ -160,7 +160,7 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, filterWithTags := false for _, filter := range view.Filters { switch filter.Field { - case "account", "resource", "provider", "name", "region": + case "account", "resource", "service", "provider", "name", "region": query, err := generateStandardFilterQuery(filter, false) if err != nil { return nil, err @@ -178,13 +178,6 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, return nil, err } whereQueries = append(whereQueries, query) - case "tags:": - filterWithTags = true - query, err := generateStandardFilterQuery(filter, true) - if err != nil { - return nil, err - } - whereQueries = append(whereQueries, query) case "tags": query, err := generateEmptyFilterQuery(filter) if err != nil { @@ -192,7 +185,16 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, } whereQueries = append(whereQueries, query) default: - return nil, fmt.Errorf("unsupported field: %s", filter.Field) + if strings.HasPrefix(filter.Field, "tag:") { + filterWithTags = true + query, err := generateStandardFilterQuery(filter, true) + if err != nil { + return nil, err + } + whereQueries = append(whereQueries, query) + } else { + return nil, fmt.Errorf("unsupported field: %s", filter.Field) + } } } @@ -200,6 +202,18 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, return queryBuilderWithFilter(view, queryTitle, arguments, queryParameter, filterWithTags, whereClause), nil } +func (repo *Repository) UpdateQuery(query, queryTitle string) error { + obj, exists := repo.queries[queryTitle] + if !exists { + return fmt.Errorf("queryTitle %s not found in repository", queryTitle) + } + + obj.Query = query + repo.queries[queryTitle] = obj + + return nil +} + func queryBuilderWithFilter(view models.View, queryTitle string, arguments []int64, query string, withTags bool, whereClause string) []string { searchQuery := []string{} limit, skip := arguments[0], arguments[1] @@ -267,18 +281,21 @@ func generateEmptyFilterQuery(filter models.Filter) (string, error) { } func generateStandardFilterQuery(filter models.Filter, withTag bool) (string, error) { - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } key := strings.ReplaceAll(filter.Field, "tag:", "") switch filter.Operator { case "IS": + for i := 0; i < len(filter.Values); i++ { + filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) + } if withTag { query := fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') IN (%s)))", key, strings.Join(filter.Values, ",")) return query, nil } return fmt.Sprintf("(%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")), nil case "IS_NOT": + for i := 0; i < len(filter.Values); i++ { + filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) + } if withTag { query := fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') NOT IN (%s)))", key, strings.Join(filter.Values, ",")) return query, nil @@ -289,7 +306,7 @@ func generateStandardFilterQuery(filter models.Filter, withTag bool) (string, er specialChar := "%" for i := 0; i < len(filter.Values); i++ { queries = append(queries, fmt.Sprintf("(%s LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } + } return fmt.Sprintf("(%s)", strings.Join(queries, " OR ")), nil case "NOT_CONTAINS": queries := make([]string, 0) From ee420b13582a9fb6c527d2a74ed876e042be0540 Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Sun, 19 May 2024 16:31:31 +0530 Subject: [PATCH 4/9] feat: added relation and stats with filters to controller --- handlers/resources_handler.go | 10 ++ handlers/stats_handler.go | 259 ++------------------------------ repository/postgres/postgres.go | 13 +- repository/sqlite/sqlite.go | 21 ++- 4 files changed, 53 insertions(+), 250 deletions(-) diff --git a/handlers/resources_handler.go b/handlers/resources_handler.go index ea5badd74..69793ef09 100644 --- a/handlers/resources_handler.go +++ b/handlers/resources_handler.go @@ -108,6 +108,9 @@ func (handler *ApiHandler) RelationStatsHandler(c *gin.Context) { return } + view := new(models.View) + view.Filters = filters + whereQueries := make([]string, 0) for _, filter := range filters { if filter.Field == "region" || filter.Field == "service" || filter.Field == "provider" { @@ -213,6 +216,13 @@ func (handler *ApiHandler) RelationStatsHandler(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + + output, err = handler.ctrl.RelationWithFilter(c, *view, []int64{}, "") + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + out := make([]models.OutputRelationResponse, 0) for _, ele := range output { out = append(out, models.OutputRelationResponse{ diff --git a/handlers/stats_handler.go b/handlers/stats_handler.go index 0e945ceb3..418c4262d 100644 --- a/handlers/stats_handler.go +++ b/handlers/stats_handler.go @@ -2,15 +2,11 @@ package handlers import ( "encoding/json" - "fmt" "net/http" - "strconv" - "strings" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/tailwarden/komiser/models" - "github.com/uptrace/bun/dialect" ) func (handler *ApiHandler) StatsHandler(c *gin.Context) { @@ -59,248 +55,25 @@ func (handler *ApiHandler) FilterStatsHandler(c *gin.Context) { return } - filterWithTags := false - whereQueries := make([]string, 0) - for _, filter := range filters { - if filter.Field == "name" || filter.Field == "region" || filter.Field == "service" || filter.Field == "provider" || filter.Field == "account" { - switch filter.Operator { - case "IS": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("(%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")) - whereQueries = append(whereQueries, query) - case "IS_NOT": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("(%s NOT IN (%s))", filter.Field, strings.Join(filter.Values, ",")) - whereQueries = append(whereQueries, query) - case "CONTAINS": - queries := make([]string, 0) - specialChar := "%" - for i := 0; i < len(filter.Values); i++ { - queries = append(queries, fmt.Sprintf("(%s LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } - whereQueries = append(whereQueries, fmt.Sprintf("(%s)", strings.Join(queries, " OR "))) - case "NOT_CONTAINS": - queries := make([]string, 0) - specialChar := "%" - for i := 0; i < len(filter.Values); i++ { - queries = append(queries, fmt.Sprintf("(%s NOT LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } - whereQueries = append(whereQueries, fmt.Sprintf("(%s)", strings.Join(queries, " AND "))) - case "IS_EMPTY": - whereQueries = append(whereQueries, fmt.Sprintf("((coalesce(%s, '') = ''))", filter.Field)) - case "IS_NOT_EMPTY": - whereQueries = append(whereQueries, fmt.Sprintf("((coalesce(%s, '') != ''))", filter.Field)) - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } else if strings.HasPrefix(filter.Field, "tag:") { - filterWithTags = true - key := strings.ReplaceAll(filter.Field, "tag:", "") - switch filter.Operator { - case "CONTAINS": - case "IS": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' IN (%s)))", key, strings.Join(filter.Values, ",")) - if handler.db.Dialect().Name() == dialect.SQLite { - query = fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') IN (%s)))", key, strings.Join(filter.Values, ",")) - } - whereQueries = append(whereQueries, query) - case "NOT_CONTAINS": - case "IS_NOT": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' NOT IN (%s)))", key, strings.Join(filter.Values, ",")) - if handler.db.Dialect().Name() == dialect.SQLite { - query = fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') NOT IN (%s)))", key, strings.Join(filter.Values, ",")) - } - whereQueries = append(whereQueries, query) - case "IS_EMPTY": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') = ''))", key)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' = ''))", key)) - } - case "IS_NOT_EMPTY": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("((json_extract(value, '$.key') = '%s') AND (json_extract(value, '$.value') != ''))", key)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("((res->>'key' = '%s') AND (res->>'value' != ''))", key)) - } - case "EXISTS": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("((json_extract(value, '$.key') = '%s'))", key)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("((res->>'key' = '%s'))", key)) - } - case "NOT_EXISTS": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf(`(NOT EXISTS (SELECT 1 FROM json_each(resources.tags) WHERE (json_extract(value, '$.key') = '%s')))`, key)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("((res->>'key' != '%s'))", key)) - } - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } else if filter.Field == "tags" { - switch filter.Operator { - case "IS_EMPTY": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, "json_array_length(tags) = 0") - } else { - whereQueries = append(whereQueries, "jsonb_array_length(tags) = 0") - } - case "IS_NOT_EMPTY": - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, "json_array_length(tags) != 0") - } else { - whereQueries = append(whereQueries, "jsonb_array_length(tags) != 0") - } - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } else if filter.Field == "cost" { - switch filter.Operator { - case "EQUAL": - cost, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - whereQueries = append(whereQueries, fmt.Sprintf("(cost = %f)", cost)) - case "BETWEEN": - min, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - max, err := strconv.ParseFloat(filter.Values[1], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - whereQueries = append(whereQueries, fmt.Sprintf("(cost >= %f AND cost <= %f)", min, max)) - case "GREATER_THAN": - cost, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - whereQueries = append(whereQueries, fmt.Sprintf("(cost > %f)", cost)) - case "LESS_THAN": - cost, err := strconv.ParseFloat(filter.Values[0], 64) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - whereQueries = append(whereQueries, fmt.Sprintf("(cost < %f)", cost)) - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } else { - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } - - whereClause := strings.Join(whereQueries, " AND ") - - if filterWithTags { - regions := struct { - Count int `bun:"count" json:"total"` - }{} - - query := fmt.Sprintf("FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s", whereClause) - if handler.db.Dialect().Name() == dialect.SQLite { - query = fmt.Sprintf("FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s", whereClause) - } - - err = handler.db.NewRaw(fmt.Sprintf("SELECT COUNT(*) as count FROM (SELECT DISTINCT region %s) AS temp", query)).Scan(handler.ctx, ®ions) - if err != nil { - logrus.WithError(err).Error("scan failed") - } - - resources := struct { - Count int `bun:"count" json:"total"` - }{} - - err = handler.db.NewRaw(fmt.Sprintf("SELECT COUNT(*) as count %s", query)).Scan(handler.ctx, &resources) - if err != nil { - logrus.WithError(err).Error("scan failed") - } + view := new(models.View) + view.Filters = filters - cost := struct { - Sum float64 `bun:"sum" json:"total"` - }{} - - err = handler.db.NewRaw(fmt.Sprintf("SELECT SUM(cost) as sum %s", query)).Scan(handler.ctx, &cost) - if err != nil { - logrus.WithError(err).Error("scan failed") - } - - output := struct { - Resources int `json:"resources"` - Regions int `json:"regions"` - Costs float64 `json:"costs"` - }{ - Resources: resources.Count, - Regions: regions.Count, - Costs: cost.Sum, - } - - c.JSON(http.StatusOK, output) + regionCount, resourceCount, costCount, err := handler.ctrl.StatsWithFilter(c, *view, []int64{}, "") + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return - } else { - query := fmt.Sprintf("FROM resources WHERE %s", whereClause) - - regions := struct { - Count int `bun:"count" json:"total"` - }{} - err = handler.db.NewRaw(fmt.Sprintf("SELECT COUNT(*) as count FROM (SELECT DISTINCT region %s) AS temp", query)).Scan(handler.ctx, ®ions) - if err != nil { - logrus.WithError(err).Error("scan failed") - } - - resources := struct { - Count int `bun:"count" json:"total"` - }{} - - err = handler.db.NewRaw(fmt.Sprintf("SELECT COUNT(*) as count %s", query)).Scan(handler.ctx, &resources) - if err != nil { - logrus.WithError(err).Error("scan failed") - } - - cost := struct { - Sum float64 `bun:"sum" json:"total"` - }{} - - err = handler.db.NewRaw(fmt.Sprintf("SELECT SUM(cost) as sum %s", query)).Scan(handler.ctx, &cost) - if err != nil { - logrus.WithError(err).Error("scan failed") - } - - output := struct { - Resources int `json:"resources"` - Regions int `json:"regions"` - Costs float64 `json:"costs"` - }{ - Resources: resources.Count, - Regions: regions.Count, - Costs: cost.Sum, - } - - c.JSON(http.StatusOK, output) } + output := struct { + Resources int `json:"resources"` + Regions int `json:"regions"` + Costs float64 `json:"costs"` + }{ + Resources: resourceCount.Count, + Regions: regionCount.Count, + Costs: costCount.Total, + } + + c.JSON(http.StatusOK, output) } func (handler *ApiHandler) ListRegionsHandler(c *gin.Context) { diff --git a/repository/postgres/postgres.go b/repository/postgres/postgres.go index 5d623363e..8876dc0b4 100644 --- a/repository/postgres/postgres.go +++ b/repository/postgres/postgres.go @@ -7,6 +7,7 @@ import ( "fmt" "strconv" "strings" + "sync" "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/repository" @@ -14,6 +15,7 @@ import ( ) type Repository struct { + mu sync.RWMutex db *bun.DB queries map[string]repository.Object } @@ -132,7 +134,9 @@ var Queries = map[string]repository.Object{ func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, schema interface{}, conditions [][3]string) (sql.Result, error) { var resp sql.Result var err error + repo.mu.RLock() query, ok := Queries[queryTitle] + repo.mu.RUnlock() if !ok { return nil, repository.ErrQueryNotFound } @@ -203,12 +207,17 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, } func (repo *Repository) UpdateQuery(query, queryTitle string) error { + + repo.mu.RLock() obj, exists := repo.queries[queryTitle] - if !exists { + repo.mu.RUnlock() + if !exists { return fmt.Errorf("queryTitle %s not found in repository", queryTitle) } - obj.Query = query + repo.mu.Lock() + obj.Query = query repo.queries[queryTitle] = obj + repo.mu.Unlock() return nil } diff --git a/repository/sqlite/sqlite.go b/repository/sqlite/sqlite.go index da8900f79..91cf690ec 100644 --- a/repository/sqlite/sqlite.go +++ b/repository/sqlite/sqlite.go @@ -7,6 +7,7 @@ import ( "fmt" "strconv" "strings" + "sync" "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/repository" @@ -14,6 +15,7 @@ import ( ) type Repository struct { + mu sync.RWMutex db *bun.DB queries map[string]repository.Object } @@ -132,11 +134,13 @@ var Queries = map[string]repository.Object{ func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, schema interface{}, conditions [][3]string) (sql.Result, error) { var resp sql.Result var err error + repo.mu.RLock() query, ok := Queries[queryTitle] + repo.mu.RUnlock() if !ok { return nil, repository.ErrQueryNotFound } - switch query.Type { + switch query.Type { case repository.RAW: err = repository.ExecuteRaw(ctx, repo.db, query.Query, schema, conditions) @@ -203,20 +207,27 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, } func (repo *Repository) UpdateQuery(query, queryTitle string) error { + + repo.mu.RLock() obj, exists := repo.queries[queryTitle] - if !exists { + repo.mu.RUnlock() + if !exists { return fmt.Errorf("queryTitle %s not found in repository", queryTitle) } - - obj.Query = query + repo.mu.Lock() + obj.Query = query repo.queries[queryTitle] = obj + repo.mu.Unlock() return nil } func queryBuilderWithFilter(view models.View, queryTitle string, arguments []int64, query string, withTags bool, whereClause string) []string { searchQuery := []string{} - limit, skip := arguments[0], arguments[1] + var limit, skip int64 + if len(arguments) >= 2 { + limit, skip = arguments[0], arguments[1] + } if len(view.Filters) == 0 { switch queryTitle { case repository.ListRelationWithFilter: From 8adfb2f6e068d0a31b3497e9894f79d87ee1988f Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Sun, 19 May 2024 16:36:03 +0530 Subject: [PATCH 5/9] fix: added missing functions in controller --- controller/controller.go | 8 ++++++++ controller/resources.go | 18 ++++++++++++++++++ controller/stats.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/controller/controller.go b/controller/controller.go index 7f1713615..12cb16711 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -11,6 +11,10 @@ type totalOutput struct { Total int `bun:"total" json:"total"` } +type resourceCountOutput struct { + Count int `bun:"count" json:"total"` +} + type costOutput struct { Total float64 `bun:"sum" json:"total"` } @@ -19,6 +23,10 @@ type regionOutput struct { Region string `bun:"region" json:"region"` } +type regionCountOuput struct { + Count int `bun:"count" json:"total"` +} + type providerOutput struct { Provider string `bun:"provider" json:"provider"` } diff --git a/controller/resources.go b/controller/resources.go index d4902248c..4c312ae8a 100644 --- a/controller/resources.go +++ b/controller/resources.go @@ -56,4 +56,22 @@ func (ctrl *Controller) ResourceWithFilter(c context.Context, view models.View, } } return +} + +func (ctrl *Controller) RelationWithFilter(c context.Context, view models.View, arguments []int64, queryParameter string) (resources []models.Resource, err error) { + resources = make([]models.Resource, 0) + queries, err := ctrl.repo.GenerateFilterQuery(view, repository.ListResourceWithFilter, arguments, queryParameter) + if err != nil { + return + } + for _, query := range queries { + if err = ctrl.repo.UpdateQuery(query, repository.ListResourceWithFilter); err != nil { + return + } + _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}) + if err != nil { + return + } + } + return } \ No newline at end of file diff --git a/controller/stats.go b/controller/stats.go index cf484fd3d..d5765e9ff 100644 --- a/controller/stats.go +++ b/controller/stats.go @@ -31,3 +31,36 @@ func (ctrl *Controller) ListAccountNames(c context.Context) (accounts []accountO _, err = ctrl.repo.HandleQuery(c, repository.ListAccountsKey, &accounts, nil) return } + +func (ctrl *Controller) StatsWithFilter(c context.Context, view models.View, arguments []int64, queryParameter string) (regionCount regionCountOuput, resourceCount resourceCountOutput, costCount costOutput, err error) { + queries, err := ctrl.repo.GenerateFilterQuery(view, repository.ListStatsWithFilter, arguments, queryParameter) + if err != nil { + return + } + if err = ctrl.repo.UpdateQuery(queries[0], repository.ListResourceWithFilter); err != nil { + return + } + _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, ®ionCount, [][3]string{}) + if err != nil { + return + } + + // for resource count + if err = ctrl.repo.UpdateQuery(queries[1], repository.ListResourceWithFilter); err != nil { + return + } + _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resourceCount, [][3]string{}) + if err != nil { + return + } + + // for cost sum + if err = ctrl.repo.UpdateQuery(queries[2], repository.ListResourceWithFilter); err != nil { + return + } + _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &costCount, [][3]string{}) + if err != nil { + return + } + return +} \ No newline at end of file From fd321c30ddd640155a0081fd03eac50a5b034f41 Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Wed, 22 May 2024 11:24:52 +0530 Subject: [PATCH 6/9] feat: added raw query support to controller --- controller/accounts.go | 12 ++++++------ controller/alerts.go | 6 +++--- controller/controller.go | 10 +--------- controller/resources.go | 19 ++++++++----------- controller/stats.go | 27 +++++++++------------------ controller/tags.go | 2 +- controller/views.go | 14 +++++++------- handlers/stats_handler.go | 4 ++-- repository/postgres/postgres.go | 20 ++++++++++++-------- repository/sqlite/sqlite.go | 20 ++++++++++++-------- 10 files changed, 61 insertions(+), 73 deletions(-) diff --git a/controller/accounts.go b/controller/accounts.go index 1bda29326..f6fd0038c 100644 --- a/controller/accounts.go +++ b/controller/accounts.go @@ -8,7 +8,7 @@ import ( ) func (ctrl *Controller) ListAccounts(c context.Context) (accounts []models.Account, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &accounts, nil) + _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &accounts, nil, "") return } @@ -20,12 +20,12 @@ func (ctrl *Controller) CountResources(c context.Context, provider, name string) if name != "" { conditions = append(conditions, [3]string{"account", "=", name}) } - _, err = ctrl.repo.HandleQuery(c, repository.ResourceCountKey, &output, conditions) + _, err = ctrl.repo.HandleQuery(c, repository.ResourceCountKey, &output, conditions, "") return } func (ctrl *Controller) InsertAccount(c context.Context, account models.Account) (lastId int64, err error) { - result, err := ctrl.repo.HandleQuery(c, repository.InsertKey, &account, nil) + result, err := ctrl.repo.HandleQuery(c, repository.InsertKey, &account, nil, "") if err != nil { return } @@ -33,7 +33,7 @@ func (ctrl *Controller) InsertAccount(c context.Context, account models.Account) } func (ctrl *Controller) RescanAccount(c context.Context, account *models.Account, accountId string) (rows int64, err error) { - res, err := ctrl.repo.HandleQuery(c, repository.ReScanAccountKey, account, [][3]string{{"id", "=", accountId}, {"status", "=", "CONNECTED"}}) + res, err := ctrl.repo.HandleQuery(c, repository.ReScanAccountKey, account, [][3]string{{"id", "=", accountId}, {"status", "=", "CONNECTED"}}, "") if err != nil { return 0, err } @@ -41,11 +41,11 @@ func (ctrl *Controller) RescanAccount(c context.Context, account *models.Account } func (ctrl *Controller) DeleteAccount(c context.Context, accountId string) (err error) { - _, err = ctrl.repo.HandleQuery(c, repository.DeleteKey, new(models.Account), [][3]string{{"id", "=", accountId}}) + _, err = ctrl.repo.HandleQuery(c, repository.DeleteKey, new(models.Account), [][3]string{{"id", "=", accountId}}, "") return } func (ctrl *Controller) UpdateAccount(c context.Context, account models.Account, accountId string) (err error) { - _, err = ctrl.repo.HandleQuery(c, repository.UpdateAccountKey, &account, [][3]string{{"id", "=", accountId}}) + _, err = ctrl.repo.HandleQuery(c, repository.UpdateAccountKey, &account, [][3]string{{"id", "=", accountId}}, "") return } diff --git a/controller/alerts.go b/controller/alerts.go index 75a96cf49..014ee96c9 100644 --- a/controller/alerts.go +++ b/controller/alerts.go @@ -8,7 +8,7 @@ import ( ) func (ctrl *Controller) InsertAlert(c context.Context, alert models.Alert) (alertId int64, err error) { - result, err := ctrl.repo.HandleQuery(c, repository.InsertKey, &alert, nil) + result, err := ctrl.repo.HandleQuery(c, repository.InsertKey, &alert, nil, "") if err != nil { return } @@ -16,11 +16,11 @@ func (ctrl *Controller) InsertAlert(c context.Context, alert models.Alert) (aler } func (ctrl *Controller) UpdateAlert(c context.Context, alert models.Alert, alertId string) (err error) { - _, err = ctrl.repo.HandleQuery(c, repository.UpdateAlertKey, &alert, [][3]string{{"id", "=", alertId}}) + _, err = ctrl.repo.HandleQuery(c, repository.UpdateAlertKey, &alert, [][3]string{{"id", "=", alertId}},"") return } func (ctrl *Controller) DeleteAlert(c context.Context, alertId string) (err error) { - _, err = ctrl.repo.HandleQuery(c, repository.DeleteKey, new(models.Alert), [][3]string{{"id", "=", alertId}}) + _, err = ctrl.repo.HandleQuery(c, repository.DeleteKey, new(models.Alert), [][3]string{{"id", "=", alertId}}, "") return } diff --git a/controller/controller.go b/controller/controller.go index 12cb16711..a8f1bc796 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -11,10 +11,6 @@ type totalOutput struct { Total int `bun:"total" json:"total"` } -type resourceCountOutput struct { - Count int `bun:"count" json:"total"` -} - type costOutput struct { Total float64 `bun:"sum" json:"total"` } @@ -23,10 +19,6 @@ type regionOutput struct { Region string `bun:"region" json:"region"` } -type regionCountOuput struct { - Count int `bun:"count" json:"total"` -} - type providerOutput struct { Provider string `bun:"provider" json:"provider"` } @@ -40,7 +32,7 @@ type accountOutput struct { } type Repository interface { - HandleQuery(context.Context, string, interface{}, [][3]string) (sql.Result, error) + HandleQuery(context.Context, string, interface{}, [][3]string, string) (sql.Result, error) GenerateFilterQuery(view models.View, queryTitle string, arguments []int64, queryParameter string) ([]string, error) UpdateQuery(query, queryTitle string) error } diff --git a/controller/resources.go b/controller/resources.go index 4c312ae8a..c16d387a2 100644 --- a/controller/resources.go +++ b/controller/resources.go @@ -9,34 +9,34 @@ import ( ) func (ctrl *Controller) GetResource(c context.Context, resourceId string) (resource models.Resource, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &resource, [][3]string{{"resource_id", "=", resourceId}}) + _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &resource, [][3]string{{"resource_id", "=", resourceId}}, "") return } func (ctrl *Controller) GetResources(c context.Context, idList string) (resources []models.Resource, err error) { resources = make([]models.Resource, 0) - _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &resources, [][3]string{{"id", "IN", "(" + strings.Trim(idList, "[]") + ")"}}) + _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &resources, [][3]string{{"id", "IN", "(" + strings.Trim(idList, "[]") + ")"}}, "") return } func (ctrl *Controller) ListResources(c context.Context) (resources []models.Resource, err error) { resources = make([]models.Resource, 0) - _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &resources, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &resources, [][3]string{}, "") return } func (ctrl *Controller) CountRegionsFromResources(c context.Context) (regions totalOutput, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.RegionResourceCountKey, ®ions, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.RegionResourceCountKey, ®ions, [][3]string{}, "") return } func (ctrl *Controller) CountRegionsFromAccounts(c context.Context) (accounts totalOutput, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.AccountsResourceCountKey, &accounts, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.AccountsResourceCountKey, &accounts, [][3]string{}, "") return } func (ctrl *Controller) SumResourceCost(c context.Context) (cost costOutput, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ResourceCostSumKey, &cost, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.ResourceCostSumKey, &cost, [][3]string{}, "") return } @@ -50,7 +50,7 @@ func (ctrl *Controller) ResourceWithFilter(c context.Context, view models.View, if err = ctrl.repo.UpdateQuery(query, repository.ListResourceWithFilter); err != nil { return } - _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}, "") if err != nil { return } @@ -65,10 +65,7 @@ func (ctrl *Controller) RelationWithFilter(c context.Context, view models.View, return } for _, query := range queries { - if err = ctrl.repo.UpdateQuery(query, repository.ListResourceWithFilter); err != nil { - return - } - _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}, query) if err != nil { return } diff --git a/controller/stats.go b/controller/stats.go index d5765e9ff..d5ceedbc0 100644 --- a/controller/stats.go +++ b/controller/stats.go @@ -8,57 +8,48 @@ import ( ) func (ctrl *Controller) LocationStatsBreakdown(c context.Context) (groups []models.OutputResources, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.LocationBreakdownStatKey, &groups, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.LocationBreakdownStatKey, &groups, [][3]string{}, "") return } func (ctrl *Controller) ListRegions(c context.Context) (regions []regionOutput, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListRegionsKey, ®ions, nil) + _, err = ctrl.repo.HandleQuery(c, repository.ListRegionsKey, ®ions, nil, "") return } func (ctrl *Controller) ListProviders(c context.Context) (providers []providerOutput, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListProvidersKey, &providers, nil) + _, err = ctrl.repo.HandleQuery(c, repository.ListProvidersKey, &providers, nil, "") return } func (ctrl *Controller) ListServices(c context.Context) (services []serviceOutput, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListServicesKey, &services, nil) + _, err = ctrl.repo.HandleQuery(c, repository.ListServicesKey, &services, nil, "") return } func (ctrl *Controller) ListAccountNames(c context.Context) (accounts []accountOutput, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListAccountsKey, &accounts, nil) + _, err = ctrl.repo.HandleQuery(c, repository.ListAccountsKey, &accounts, nil, "") return } -func (ctrl *Controller) StatsWithFilter(c context.Context, view models.View, arguments []int64, queryParameter string) (regionCount regionCountOuput, resourceCount resourceCountOutput, costCount costOutput, err error) { +func (ctrl *Controller) StatsWithFilter(c context.Context, view models.View, arguments []int64, queryParameter string) (regionCount totalOutput, resourceCount totalOutput, costCount costOutput, err error) { queries, err := ctrl.repo.GenerateFilterQuery(view, repository.ListStatsWithFilter, arguments, queryParameter) if err != nil { return } - if err = ctrl.repo.UpdateQuery(queries[0], repository.ListResourceWithFilter); err != nil { - return - } - _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, ®ionCount, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.ListStatsWithFilter, ®ionCount, [][3]string{}, queries[0]) if err != nil { return } // for resource count - if err = ctrl.repo.UpdateQuery(queries[1], repository.ListResourceWithFilter); err != nil { - return - } - _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resourceCount, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.ListStatsWithFilter, &resourceCount, [][3]string{}, queries[1]) if err != nil { return } // for cost sum - if err = ctrl.repo.UpdateQuery(queries[2], repository.ListResourceWithFilter); err != nil { - return - } - _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &costCount, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.ListStatsWithFilter, &costCount, [][3]string{}, queries[2]) if err != nil { return } diff --git a/controller/tags.go b/controller/tags.go index c2d0fb5ff..ceca17aff 100644 --- a/controller/tags.go +++ b/controller/tags.go @@ -10,6 +10,6 @@ import ( func (ctrl *Controller) UpdateTags(c context.Context, tags []models.Tag, resourceId string) (resource models.Resource, err error) { resource.Tags = tags - _, err = ctrl.repo.HandleQuery(c, repository.UpdateTagsKey, &resource, [][3]string{{"id", "=", fmt.Sprint(resourceId)}}) + _, err = ctrl.repo.HandleQuery(c, repository.UpdateTagsKey, &resource, [][3]string{{"id", "=", fmt.Sprint(resourceId)}}, "") return } diff --git a/controller/views.go b/controller/views.go index 6c26b9bf8..11d04f2d2 100644 --- a/controller/views.go +++ b/controller/views.go @@ -8,17 +8,17 @@ import ( ) func (ctrl *Controller) GetView(c context.Context, viewId string) (view models.View, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &view, [][3]string{{"id", "=", viewId}}) + _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &view, [][3]string{{"id", "=", viewId}}, "") return } func (ctrl *Controller) ListViews(c context.Context) (views []models.View, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &views, [][3]string{}) + _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &views, [][3]string{}, "") return } func (ctrl *Controller) InsertView(c context.Context, view models.View) (viewId int64, err error) { - result, err := ctrl.repo.HandleQuery(c, repository.InsertKey, &view, nil) + result, err := ctrl.repo.HandleQuery(c, repository.InsertKey, &view, nil, "") if err != nil { return } @@ -26,21 +26,21 @@ func (ctrl *Controller) InsertView(c context.Context, view models.View) (viewId } func (ctrl *Controller) UpdateView(c context.Context, view models.View, viewId string) (err error) { - _, err = ctrl.repo.HandleQuery(c, repository.UpdateViewKey, &view, [][3]string{{"id", "=", viewId}}) + _, err = ctrl.repo.HandleQuery(c, repository.UpdateViewKey, &view, [][3]string{{"id", "=", viewId}}, "") return } func (ctrl *Controller) DeleteView(c context.Context, viewId string) (err error) { - _, err = ctrl.repo.HandleQuery(c, repository.DeleteKey, new(models.View), [][3]string{{"id", "=", viewId}}) + _, err = ctrl.repo.HandleQuery(c, repository.DeleteKey, new(models.View), [][3]string{{"id", "=", viewId}}, "") return } func (ctrl *Controller) UpdateViewExclude(c context.Context, view models.View, viewId string) (err error) { - _, err = ctrl.repo.HandleQuery(c, repository.UpdateViewExcludeKey, &view, [][3]string{{"id", "=", viewId}}) + _, err = ctrl.repo.HandleQuery(c, repository.UpdateViewExcludeKey, &view, [][3]string{{"id", "=", viewId}}, "") return } func (ctrl *Controller) ListViewAlerts(c context.Context, viewId string) (alerts []models.Alert, err error) { - _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &alerts, [][3]string{{"view_id", "=", viewId}}) + _, err = ctrl.repo.HandleQuery(c, repository.ListKey, &alerts, [][3]string{{"view_id", "=", viewId}}, "") return } diff --git a/handlers/stats_handler.go b/handlers/stats_handler.go index 418c4262d..8715a07f7 100644 --- a/handlers/stats_handler.go +++ b/handlers/stats_handler.go @@ -68,8 +68,8 @@ func (handler *ApiHandler) FilterStatsHandler(c *gin.Context) { Regions int `json:"regions"` Costs float64 `json:"costs"` }{ - Resources: resourceCount.Count, - Regions: regionCount.Count, + Resources: resourceCount.Total, + Regions: regionCount.Total, Costs: costCount.Total, } diff --git a/repository/postgres/postgres.go b/repository/postgres/postgres.go index 8876dc0b4..cb7c6f568 100644 --- a/repository/postgres/postgres.go +++ b/repository/postgres/postgres.go @@ -63,7 +63,7 @@ var Queries = map[string]repository.Object{ Type: repository.RAW, }, repository.AccountsResourceCountKey: { - Query: "SELECT COUNT(*) as count FROM (SELECT DISTINCT account FROM resources) AS temp", + Query: "SELECT COUNT(*) as total FROM (SELECT DISTINCT account FROM resources) AS temp", Type: repository.RAW, }, repository.RegionResourceCountKey: { @@ -121,17 +121,17 @@ var Queries = map[string]repository.Object{ Type: repository.RAW, Query: "", Params: []string{ - "SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s) AS temp", - "SELECT COUNT(*) as count FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s", + "SELECT COUNT(*) as total FROM (SELECT DISTINCT region FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s) AS temp", + "SELECT COUNT(*) as total FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s", "SELECT SUM(cost) as sum FROM resources CROSS JOIN jsonb_array_elements(tags) AS res WHERE %s", - "SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources WHERE %s) AS temp", - "SELECT COUNT(*) as count FROM resources WHERE %s", + "SELECT COUNT(*) as total FROM (SELECT DISTINCT region FROM resources WHERE %s) AS temp", + "SELECT COUNT(*) as total FROM resources WHERE %s", "SELECT SUM(cost) as sum FROM resources WHERE %s", }, }, } -func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, schema interface{}, conditions [][3]string) (sql.Result, error) { +func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, schema interface{}, conditions [][3]string, rawQuery string) (sql.Result, error) { var resp sql.Result var err error repo.mu.RLock() @@ -142,8 +142,12 @@ func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, sche } switch query.Type { case repository.RAW: - err = repository.ExecuteRaw(ctx, repo.db, query.Query, schema, conditions) - + if rawQuery != "" && query.Query == "" { + err = repository.ExecuteRaw(ctx, repo.db, rawQuery, schema, conditions) + } else { + err = repository.ExecuteRaw(ctx, repo.db, query.Query, schema, conditions) + } + case repository.SELECT: err = repository.ExecuteSelect(ctx, repo.db, schema, conditions) diff --git a/repository/sqlite/sqlite.go b/repository/sqlite/sqlite.go index 91cf690ec..8af1b1866 100644 --- a/repository/sqlite/sqlite.go +++ b/repository/sqlite/sqlite.go @@ -63,11 +63,11 @@ var Queries = map[string]repository.Object{ Type: repository.RAW, }, repository.AccountsResourceCountKey: { - Query: "SELECT COUNT(*) as count FROM (SELECT DISTINCT account FROM resources) AS temp", + Query: "SELECT COUNT(*) as total FROM (SELECT DISTINCT account FROM resources)", Type: repository.RAW, }, repository.RegionResourceCountKey: { - Query: "SELECT COUNT(*) as total FROM (SELECT DISTINCT region FROM resources) AS temp", + Query: "SELECT COUNT(*) as total FROM (SELECT DISTINCT region FROM resources)", Type: repository.RAW, }, repository.FilterResourceCountKey: { @@ -121,17 +121,17 @@ var Queries = map[string]repository.Object{ Type: repository.RAW, Query: "", Params: []string{ - "SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s) AS temp", - "SELECT COUNT(*) as count FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s", + "SELECT COUNT(*) as total FROM (SELECT DISTINCT region FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s) AS temp", + "SELECT COUNT(*) as total FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s", "SELECT SUM(cost) as sum FROM resources CROSS JOIN json_each(tags) WHERE type='object' AND %s", - "SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources WHERE %s) AS temp", - "SELECT COUNT(*) as count FROM resources WHERE %s", + "SELECT COUNT(*) as total FROM (SELECT DISTINCT region FROM resources WHERE %s) AS temp", + "SELECT COUNT(*) as total FROM resources WHERE %s", "SELECT SUM(cost) as sum FROM resources WHERE %s", }, }, } -func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, schema interface{}, conditions [][3]string) (sql.Result, error) { +func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, schema interface{}, conditions [][3]string, rawQuery string) (sql.Result, error) { var resp sql.Result var err error repo.mu.RLock() @@ -142,7 +142,11 @@ func (repo *Repository) HandleQuery(ctx context.Context, queryTitle string, sche } switch query.Type { case repository.RAW: - err = repository.ExecuteRaw(ctx, repo.db, query.Query, schema, conditions) + if rawQuery != "" && query.Query == "" { + err = repository.ExecuteRaw(ctx, repo.db, rawQuery, schema, conditions) + } else { + err = repository.ExecuteRaw(ctx, repo.db, query.Query, schema, conditions) + } case repository.SELECT: err = repository.ExecuteSelect(ctx, repo.db, schema, conditions) From 3bdda9a70a9b5e914d519d83839981e70812946d Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Wed, 22 May 2024 14:43:03 +0530 Subject: [PATCH 7/9] fix: refactor --- controller/controller.go | 1 - controller/resources.go | 5 +---- repository/postgres/postgres.go | 16 ---------------- repository/sqlite/sqlite.go | 16 ---------------- 4 files changed, 1 insertion(+), 37 deletions(-) diff --git a/controller/controller.go b/controller/controller.go index a8f1bc796..a7ce38ecc 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -34,7 +34,6 @@ type accountOutput struct { type Repository interface { HandleQuery(context.Context, string, interface{}, [][3]string, string) (sql.Result, error) GenerateFilterQuery(view models.View, queryTitle string, arguments []int64, queryParameter string) ([]string, error) - UpdateQuery(query, queryTitle string) error } type Controller struct { diff --git a/controller/resources.go b/controller/resources.go index c16d387a2..351f838a6 100644 --- a/controller/resources.go +++ b/controller/resources.go @@ -47,10 +47,7 @@ func (ctrl *Controller) ResourceWithFilter(c context.Context, view models.View, return } for _, query := range queries { - if err = ctrl.repo.UpdateQuery(query, repository.ListResourceWithFilter); err != nil { - return - } - _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}, "") + _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}, query) if err != nil { return } diff --git a/repository/postgres/postgres.go b/repository/postgres/postgres.go index cb7c6f568..444dcbb16 100644 --- a/repository/postgres/postgres.go +++ b/repository/postgres/postgres.go @@ -210,22 +210,6 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, return queryBuilderWithFilter(view, queryTitle, arguments, queryParameter, filterWithTags, whereClause), nil } -func (repo *Repository) UpdateQuery(query, queryTitle string) error { - - repo.mu.RLock() - obj, exists := repo.queries[queryTitle] - repo.mu.RUnlock() - if !exists { - return fmt.Errorf("queryTitle %s not found in repository", queryTitle) - } - repo.mu.Lock() - obj.Query = query - repo.queries[queryTitle] = obj - repo.mu.Unlock() - - return nil -} - func queryBuilderWithFilter(view models.View, queryTitle string, arguments []int64, query string, withTags bool, whereClause string) []string { searchQuery := []string{} limit, skip := arguments[0], arguments[1] diff --git a/repository/sqlite/sqlite.go b/repository/sqlite/sqlite.go index 8af1b1866..dd986644b 100644 --- a/repository/sqlite/sqlite.go +++ b/repository/sqlite/sqlite.go @@ -210,22 +210,6 @@ func (repo *Repository) GenerateFilterQuery(view models.View, queryTitle string, return queryBuilderWithFilter(view, queryTitle, arguments, queryParameter, filterWithTags, whereClause), nil } -func (repo *Repository) UpdateQuery(query, queryTitle string) error { - - repo.mu.RLock() - obj, exists := repo.queries[queryTitle] - repo.mu.RUnlock() - if !exists { - return fmt.Errorf("queryTitle %s not found in repository", queryTitle) - } - repo.mu.Lock() - obj.Query = query - repo.queries[queryTitle] = obj - repo.mu.Unlock() - - return nil -} - func queryBuilderWithFilter(view models.View, queryTitle string, arguments []int64, query string, withTags bool, whereClause string) []string { searchQuery := []string{} var limit, skip int64 From 3d7d1ce9902610087852ea40443c2f87c80174db Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Thu, 23 May 2024 11:46:45 +0530 Subject: [PATCH 8/9] feat: added relation filter support --- controller/resources.go | 4 +- handlers/resources_handler.go | 112 +--------------------------------- 2 files changed, 4 insertions(+), 112 deletions(-) diff --git a/controller/resources.go b/controller/resources.go index 351f838a6..924ed48fd 100644 --- a/controller/resources.go +++ b/controller/resources.go @@ -57,12 +57,12 @@ func (ctrl *Controller) ResourceWithFilter(c context.Context, view models.View, func (ctrl *Controller) RelationWithFilter(c context.Context, view models.View, arguments []int64, queryParameter string) (resources []models.Resource, err error) { resources = make([]models.Resource, 0) - queries, err := ctrl.repo.GenerateFilterQuery(view, repository.ListResourceWithFilter, arguments, queryParameter) + queries, err := ctrl.repo.GenerateFilterQuery(view, repository.ListRelationWithFilter, arguments, queryParameter) if err != nil { return } for _, query := range queries { - _, err = ctrl.repo.HandleQuery(c, repository.ListResourceWithFilter, &resources, [][3]string{}, query) + _, err = ctrl.repo.HandleQuery(c, repository.ListRelationWithFilter, &resources, [][3]string{}, query) if err != nil { return } diff --git a/handlers/resources_handler.go b/handlers/resources_handler.go index 69793ef09..baf82458e 100644 --- a/handlers/resources_handler.go +++ b/handlers/resources_handler.go @@ -3,10 +3,8 @@ package handlers import ( "context" "encoding/json" - "fmt" "net/http" "strconv" - "strings" "github.com/gin-gonic/gin" "github.com/tailwarden/komiser/controller" @@ -109,115 +107,9 @@ func (handler *ApiHandler) RelationStatsHandler(c *gin.Context) { } view := new(models.View) - view.Filters = filters - - whereQueries := make([]string, 0) - for _, filter := range filters { - if filter.Field == "region" || filter.Field == "service" || filter.Field == "provider" { - switch filter.Operator { - case "IS": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("(resources.%s IN (%s))", filter.Field, strings.Join(filter.Values, ",")) - whereQueries = append(whereQueries, query) - case "IS_NOT": - for i := 0; i < len(filter.Values); i++ { - filter.Values[i] = fmt.Sprintf("'%s'", filter.Values[i]) - } - query := fmt.Sprintf("(resources.%s NOT IN (%s))", filter.Field, strings.Join(filter.Values, ",")) - whereQueries = append(whereQueries, query) - case "CONTAINS": - queries := make([]string, 0) - specialChar := "%" - for i := 0; i < len(filter.Values); i++ { - queries = append(queries, fmt.Sprintf("(resources.%s LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } - whereQueries = append(whereQueries, fmt.Sprintf("(%s)", strings.Join(queries, " OR "))) - case "NOT_CONTAINS": - queries := make([]string, 0) - specialChar := "%" - for i := 0; i < len(filter.Values); i++ { - queries = append(queries, fmt.Sprintf("(resources.%s NOT LIKE '%s%s%s')", filter.Field, specialChar, filter.Values[i], specialChar)) - } - whereQueries = append(whereQueries, fmt.Sprintf("(%s)", strings.Join(queries, " AND "))) - case "IS_EMPTY": - whereQueries = append(whereQueries, fmt.Sprintf("((coalesce(resources.%s, '') = ''))", filter.Field)) - case "IS_NOT_EMPTY": - whereQueries = append(whereQueries, fmt.Sprintf("((coalesce(resources.%s, '') != ''))", filter.Field)) - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "operation is invalid or not supported"}) - return - } - } else if filter.Field == "relations" { - switch filter.Operator { - case "EQUAL": - relations, err := strconv.Atoi(filter.Values[0]) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("json_array_length(resources.relations) = %d", relations)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("jsonb_array_length(resources.relations) = %d", relations)) - } - case "GREATER_THAN": - relations, err := strconv.Atoi(filter.Values[0]) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("json_array_length(resources.relations) > %d", relations)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("jsonb_array_length(resources.relations) > %d", relations)) - } - case "LESS_THAN": - relations, err := strconv.Atoi(filter.Values[0]) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - if handler.db.Dialect().Name() == dialect.SQLite { - whereQueries = append(whereQueries, fmt.Sprintf("json_array_length(resources.relations) < %d", relations)) - } else { - whereQueries = append(whereQueries, fmt.Sprintf("jsonb_array_length(resources.relations) < %d", relations)) - } - default: - c.JSON(http.StatusInternalServerError, gin.H{"error": "value should be a number"}) - return - } - } else { - c.JSON(http.StatusInternalServerError, gin.H{"error": "field is invalid or not supported"}) - return - } - } - - whereClause := strings.Join(whereQueries, " AND ") - - output := make([]models.Resource, 0) - - query := "" - if len(filters) == 0 { - query = "SELECT DISTINCT resources.resource_id, resources.provider, resources.name, resources.service, resources.relations FROM resources WHERE (jsonb_array_length(relations) > 0)" - if handler.db.Dialect().Name() == dialect.SQLite { - query = "SELECT DISTINCT resources.resource_id, resources.provider, resources.name, resources.service, resources.relations FROM resources WHERE (json_array_length(relations) > 0)" - } - } else { - query = "SELECT DISTINCT resources.resource_id, resources.provider, resources.name, resources.service, resources.relations FROM resources WHERE (jsonb_array_length(relations) > 0) AND " + whereClause - if handler.db.Dialect().Name() == dialect.SQLite { - query = "SELECT DISTINCT resources.resource_id, resources.provider, resources.name, resources.service, resources.relations FROM resources WHERE (json_array_length(relations) > 0) AND " + whereClause - } - } - - err = handler.db.NewRaw(query).Scan(handler.ctx, &output) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } + view.Filters = filters - output, err = handler.ctrl.RelationWithFilter(c, *view, []int64{}, "") + output, err := handler.ctrl.RelationWithFilter(c, *view, []int64{}, "") if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return From da961399b39f973dff65e5ccabec2029463b096e Mon Sep 17 00:00:00 2001 From: AvineshTripathi Date: Thu, 23 May 2024 12:28:44 +0530 Subject: [PATCH 9/9] fix: lint error --- handlers/resources_handler.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/handlers/resources_handler.go b/handlers/resources_handler.go index baf82458e..719c21726 100644 --- a/handlers/resources_handler.go +++ b/handlers/resources_handler.go @@ -50,7 +50,6 @@ func NewApiHandler(ctx context.Context, telemetry bool, analytics utils.Analytic func (handler *ApiHandler) FilterResourcesHandler(c *gin.Context) { var filters []models.Filter - resources := make([]models.Resource, 0) limitRaw := c.Query("limit") skipRaw := c.Query("skip") @@ -89,7 +88,7 @@ func (handler *ApiHandler) FilterResourcesHandler(c *gin.Context) { return } view.Filters = filters - resources, err = handler.ctrl.ResourceWithFilter(c, *view, []int64{limit, skip}, queryParameter) + resources, err := handler.ctrl.ResourceWithFilter(c, *view, []int64{limit, skip}, queryParameter) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return