Merge pull request #1415 from tailwarden/develop

v3.1.16 release 🚀

dashboard/package.json

@@ -40,11 +40,11 @@
     "typescript": "5.2.2"
   },
   "devDependencies": {
-    "@commitlint/cli": "^19.2.1",
+    "@commitlint/cli": "^19.3.0",
     "@commitlint/config-conventional": "^17.2.0",
     "@jest/globals": "^29.5.0",
     "@storybook/addon-essentials": "^8.0.5",
-    "@storybook/addon-interactions": "^7.6.17",
+    "@storybook/addon-interactions": "^8.0.9",
     "@storybook/addon-links": "^8.0.4",
     "@storybook/addons": "^7.4.6",
     "@storybook/blocks": "^7.6.4",

go.mod

@@ -205,12 +205,12 @@ require (
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/mod v0.13.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect

go.sum

@@ -538,8 +538,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
@@ -567,8 +567,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ=
 golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA=
@@ -604,12 +604,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

handlers/accounts_handler.go

@@ -1,7 +1,6 @@
 package handlers
 
 import (
-	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
@@ -12,6 +11,7 @@ import (
 	"github.com/go-co-op/gocron"
 	log "github.com/sirupsen/logrus"
 	"github.com/tailwarden/komiser/models"
+	"github.com/tailwarden/komiser/repository"
 	"github.com/tailwarden/komiser/utils"
 	"github.com/uptrace/bun"
 	"github.com/uptrace/bun/dialect/pgdialect"
@@ -38,7 +38,8 @@ func (handler *ApiHandler) IsOnboardedHandler(c *gin.Context) {
 	}
 
 	accounts := make([]models.Account, 0)
-	err := handler.db.NewRaw("SELECT * FROM accounts").Scan(handler.ctx, &accounts)
+
+	_, err := handler.repo.HandleQuery(c, repository.ListKey, &accounts, nil)
 	if err != nil {
 		log.WithError(err).Error("scan failed")
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "scan failed"})
@@ -62,7 +63,7 @@ func (handler *ApiHandler) ListCloudAccountsHandler(c *gin.Context) {
 		return
 	}
 
-	err := handler.db.NewRaw("SELECT * FROM accounts").Scan(handler.ctx, &accounts)
+	_, err := handler.repo.HandleQuery(c, repository.ListKey, &accounts, nil)
 	if err != nil {
 		log.WithError(err).Error("scan failed")
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "scan failed"})
@@ -73,8 +74,10 @@ func (handler *ApiHandler) ListCloudAccountsHandler(c *gin.Context) {
 		output := struct {
 			Total int `bun:"total" json:"total"`
 		}{}
-		err = handler.db.NewRaw(fmt.Sprintf("SELECT COUNT(*) as total FROM resources WHERE provider='%s' AND account='%s'", account.Provider, account.Name)).Scan(handler.ctx, &output)
+
+		_, err := handler.repo.HandleQuery(c, repository.ResourceCountKey, &output, [][3]string{{"provider", "=", account.Provider}, {"account", "=", account.Name}})
 		if err != nil {
+			fmt.Println(err)
 			c.JSON(http.StatusInternalServerError, gin.H{"error": "scan failed"})
 			return
 		}
@@ -104,12 +107,12 @@ func (handler *ApiHandler) NewCloudAccountHandler(c *gin.Context) {
 
 		unsavedAccounts = append(unsavedAccounts, account)
 	} else {
-		result, err := handler.db.NewInsert().Model(&account).Exec(context.Background())
+
+		result, err := handler.repo.HandleQuery(c, repository.InsertKey, &account, nil)
 		if err != nil {
 			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 			return
 		}
-
 		accountId, _ := result.LastInsertId()
 		account.Id = accountId
 
@@ -152,7 +155,8 @@ func (handler *ApiHandler) ReScanAccount(c *gin.Context) {
 	accountId := c.Param("id")
 
 	account := new(models.Account)
-	res, err := handler.db.NewUpdate().Model(account).Set("status = ? ", "SCANNING").Where("id = ?", accountId).Where("status = ?", "CONNECTED").Returning("*").Exec(handler.ctx)
+	account.Status = "SCANNING"
+	res, err := handler.repo.HandleQuery(c, repository.ReScanAccountKey, account, [][3]string{{"id", "=", accountId}, {"status", "=", "CONNECTED"}})
 	if err != nil {
 		log.Error("Couldn't set status", err)
 		return
@@ -169,7 +173,7 @@ func (handler *ApiHandler) DeleteCloudAccountHandler(c *gin.Context) {
 	accountId := c.Param("id")
 
 	account := new(models.Account)
-	_, err := handler.db.NewDelete().Model(account).Where("id = ?", accountId).Exec(handler.ctx)
+	_, err := handler.repo.HandleQuery(c, repository.DeleteKey, account, [][3]string{{"id", "=", accountId}})
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
@@ -188,7 +192,7 @@ func (handler *ApiHandler) UpdateCloudAccountHandler(c *gin.Context) {
 		return
 	}
 
-	_, err = handler.db.NewUpdate().Model(&account).Column("name", "provider", "credentials").Where("id = ?", accountId).Exec(handler.ctx)
+	_, err = handler.repo.HandleQuery(c, repository.UpdateAccountKey, &account, [][3]string{{"id", "=", accountId}})
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return

handlers/alerts_handler.go

@@ -2,13 +2,14 @@ package handlers
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/tailwarden/komiser/models"
+	"github.com/tailwarden/komiser/repository"
 )
 
 func (handler *ApiHandler) IsSlackEnabledHandler(c *gin.Context) {
@@ -33,7 +34,7 @@ func (handler *ApiHandler) NewAlertHandler(c *gin.Context) {
 		return
 	}
 
-	result, err := handler.db.NewInsert().Model(&alert).Exec(context.Background())
+	result, err := handler.repo.HandleQuery(c, repository.InsertKey, &alert, nil)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
@@ -62,7 +63,7 @@ func (handler *ApiHandler) UpdateAlertHandler(c *gin.Context) {
 		return
 	}
 
-	_, err = handler.db.NewUpdate().Model(&alert).Column("name", "type", "budget", "usage", "endpoint", "secret").Where("id = ?", alertId).Exec(handler.ctx)
+	_, err = handler.repo.HandleQuery(c, repository.UpdateAlertKey, &alert, [][3]string{{"id", "=", fmt.Sprint(alertId)}})
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
@@ -75,7 +76,7 @@ func (handler *ApiHandler) DeleteAlertHandler(c *gin.Context) {
 	alertId := c.Param("id")
 
 	alert := new(models.Alert)
-	_, err := handler.db.NewDelete().Model(alert).Where("id = ?", alertId).Exec(handler.ctx)
+	_, err := handler.repo.HandleQuery(c, repository.DeleteKey, alert, [][3]string{{"id", "=", fmt.Sprint(alertId)}})
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return

handlers/csv_handler.go

@@ -14,12 +14,13 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/tailwarden/komiser/models"
+	"github.com/tailwarden/komiser/repository"
 	"github.com/uptrace/bun/dialect"
 )
 
 func (handler *ApiHandler) DownloadInventoryCSV(c *gin.Context) {
 	resources := make([]models.Resource, 0)
-	err := handler.db.NewSelect().Table("resources").Scan(handler.ctx, &resources)
+	_, err := handler.repo.HandleQuery(c, repository.ListKey, &resources, [][3]string{})
 	if err != nil {
 		logrus.WithError(err).Error("Could not read from DB")
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "cloud not read from DB"})
@@ -37,7 +38,7 @@ func (handler *ApiHandler) DownloadInventoryCSVForView(c *gin.Context) {
 	viewId := c.Param("viewId")
 
 	view := new(models.View)
-	err := handler.db.NewSelect().Model(view).Where("id = ?", viewId).Scan(handler.ctx)
+	_, err := handler.repo.HandleQuery(c, repository.ListKey, view, [][3]string{{"id", "=", fmt.Sprint(viewId)}})
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return

handlers/dashboard_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/tailwarden/komiser/models"
+	"github.com/tailwarden/komiser/repository"
 	"github.com/tailwarden/komiser/utils"
 	"github.com/uptrace/bun"
 )
@@ -32,16 +33,16 @@ func (handler *ApiHandler) DashboardStatsHandler(c *gin.Context) {
 		Count int `bun:"count" json:"total"`
 	}{}
 
-	err := handler.db.NewRaw("SELECT COUNT(*) as count FROM (SELECT DISTINCT region FROM resources) AS temp").Scan(handler.ctx, &regions)
+	_, err := handler.repo.HandleQuery(c, repository.RegionResourceCountKey, &regions, [][3]string{})
 	if err != nil {
 		logrus.WithError(err).Error("scan failed")
 	}
 
 	resources := struct {
-		Count int `bun:"count" json:"total"`
+		Count int `bun:"total" json:"total"`
 	}{}
 
-	err = handler.db.NewRaw("SELECT COUNT(*) as count FROM resources").Scan(handler.ctx, &resources)
+	_, err = handler.repo.HandleQuery(c, repository.ResourceCountKey, &resources, [][3]string{})
 	if err != nil {
 		logrus.WithError(err).Error("scan failed")
 	}
@@ -50,7 +51,7 @@ func (handler *ApiHandler) DashboardStatsHandler(c *gin.Context) {
 		Sum float64 `bun:"sum" json:"total"`
 	}{}
 
-	err = handler.db.NewRaw("SELECT SUM(cost) as sum FROM resources").Scan(handler.ctx, &cost)
+	_, err = handler.repo.HandleQuery(c, repository.ResourceCostSumKey, &cost, [][3]string{})
 	if err != nil {
 		logrus.WithError(err).Error("scan failed")
 	}
@@ -59,7 +60,7 @@ func (handler *ApiHandler) DashboardStatsHandler(c *gin.Context) {
 		Count int `bun:"count" json:"total"`
 	}{}
 
-	err = handler.db.NewRaw("SELECT COUNT(*) as count FROM (SELECT DISTINCT account FROM resources) AS temp").Scan(handler.ctx, &accounts)
+	_, err = handler.repo.HandleQuery(c, repository.RegionResourceCountKey, &regions, [][3]string{})
 	if err != nil {
 		logrus.WithError(err).Error("scan failed")
 	}
@@ -143,8 +144,7 @@ func (handler *ApiHandler) LocationBreakdownStatsHandler(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, []models.OutputLocations{})
 		return
 	}
-
-	err := handler.db.NewRaw("SELECT region as label, COUNT(*) as total FROM resources GROUP BY region ORDER by total desc;").Scan(handler.ctx, &groups)
+	_, err := handler.repo.HandleQuery(c, repository.LocationBreakdownStatKey, &groups, [][3]string{})
 	if err != nil {
 		logrus.WithError(err).Error("scan failed")
 	}

handlers/resources_handler.go

@@ -2,6 +2,7 @@ package handlers
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -11,44 +12,49 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/tailwarden/komiser/models"
-	. "github.com/tailwarden/komiser/models"
+	"github.com/tailwarden/komiser/repository"
 	"github.com/tailwarden/komiser/utils"
 	"github.com/uptrace/bun"
 	"github.com/uptrace/bun/dialect"
 )
 
 type ApiHandler struct {
-	db        *bun.DB
-	ctx       context.Context
-	telemetry bool
-	cfg       models.Config
+	db         *bun.DB
+	repo       Repository
+	ctx        context.Context
+	telemetry  bool
+	cfg        models.Config
 	configPath string
-	analytics utils.Analytics
-	accounts  []models.Account
+	analytics  utils.Analytics
+	accounts   []models.Account
+}
+
+type Repository interface {
+	HandleQuery(context.Context, repository.QueryType, interface{}, [][3]string) (sql.Result, error)
 }
 
 func NewApiHandler(ctx context.Context, telemetry bool, analytics utils.Analytics, db *bun.DB, cfg models.Config, configPath string, accounts []models.Account) *ApiHandler {
 	handler := ApiHandler{
-		db:        db,
-		ctx:       ctx,
-		telemetry: telemetry,
-		cfg:       cfg,
+		db:         db,
+		ctx:        ctx,
+		telemetry:  telemetry,
+		cfg:        cfg,
 		configPath: configPath,
-		analytics: analytics,
-		accounts:  accounts,
+		analytics:  analytics,
+		accounts:   accounts,
 	}
 	return &handler
 }
 
 func (handler *ApiHandler) FilterResourcesHandler(c *gin.Context) {
-	var filters []Filter
+	var filters []models.Filter
 
 	limitRaw := c.Query("limit")
 	skipRaw := c.Query("skip")
 	query := c.Query("query")
 	viewId := c.Query("view")
 
-	view := new(View)
+	view := new(models.View)
 	if viewId != "" {
 		err := handler.db.NewSelect().Model(view).Where("id = ?", viewId).Scan(handler.ctx)
 		if err != nil {
@@ -240,7 +246,7 @@ func (handler *ApiHandler) FilterResourcesHandler(c *gin.Context) {
 
 	whereClause := strings.Join(whereQueries, " AND ")
 
-	resources := make([]Resource, 0)
+	resources := make([]models.Resource, 0)
 
 	if len(filters) == 0 {
 		if len(query) > 0 {
@@ -305,7 +311,7 @@ func (handler *ApiHandler) FilterResourcesHandler(c *gin.Context) {
 }
 
 func (handler *ApiHandler) RelationStatsHandler(c *gin.Context) {
-	var filters []Filter
+	var filters []models.Filter
 
 	err := json.NewDecoder(c.Request.Body).Decode(&filters)
 	if err != nil {
@@ -428,17 +434,17 @@ func (handler *ApiHandler) RelationStatsHandler(c *gin.Context) {
 			Provider:   ele.Provider,
 		})
 	}
-	
+
 	c.JSON(http.StatusOK, out)
 
 }
 
 func (handler *ApiHandler) GetResourceByIdHandler(c *gin.Context) {
 	resourceId := c.Query("resourceId")
 
-	var resource Resource
+	var resource models.Resource
 
-	err := handler.db.NewSelect().Model(&resource).Where("resource_id = ?", resourceId).Scan(handler.ctx)
+	_, err := handler.repo.HandleQuery(c, repository.ListKey, &resource, [][3]string{{"resource_id", "=", resourceId}})
 	if err != nil {
 		c.JSON(http.StatusNotFound, gin.H{"error": "Resource not found"})
 	}