From 237dd7646f2a838557d587ceea5b349855bf5d7e Mon Sep 17 00:00:00 2001 From: Mohamed Labouardy Date: Tue, 19 Sep 2023 21:52:26 +0300 Subject: [PATCH 1/5] feat: expose is_onboarded and get cloud_accounts endpoints --- cmd/start.go | 2 + handlers/accounts_handler.go | 43 +++++++++++ internal/api/v1/endpoints.go | 3 + internal/config/load.go | 135 +++++++++++++++++++++++++++++++++-- internal/internal.go | 18 ++++- models/account.go | 8 +++ 6 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 handlers/accounts_handler.go create mode 100644 models/account.go diff --git a/cmd/start.go b/cmd/start.go index 830d6ed75..4f3f2a57a 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -2,6 +2,7 @@ package cmd import ( "errors" + "fmt" "time" "github.com/getsentry/sentry-go" @@ -61,6 +62,7 @@ var startCmd = &cobra.Command{ err = internal.Exec(address, port, file, telemetry, analytics, regions, cmd) if err != nil { + fmt.Println(err) return err } diff --git a/handlers/accounts_handler.go b/handlers/accounts_handler.go new file mode 100644 index 000000000..da814aa70 --- /dev/null +++ b/handlers/accounts_handler.go @@ -0,0 +1,43 @@ +package handlers + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "github.com/tailwarden/komiser/models" +) + +func (handler *ApiHandler) IsOnboardedHandler(c *gin.Context) { + output := struct { + Onboarded bool `json:"onboarded"` + }{ + Onboarded: false, + } + + accounts := make([]models.Account, 0) + err := handler.db.NewRaw("SELECT * FROM accounts").Scan(handler.ctx, &accounts) + if err != nil { + logrus.WithError(err).Error("scan failed") + c.JSON(http.StatusInternalServerError, gin.H{"error": "scan failed"}) + return + } + + if len(accounts) > 0 { + output.Onboarded = true + } + + c.JSON(http.StatusOK, output) +} + +func (handler *ApiHandler) ListCloudAccountsHandler(c *gin.Context) { + accounts := make([]models.Account, 0) + err := handler.db.NewRaw("SELECT * FROM accounts").Scan(handler.ctx, &accounts) + if err != nil { + logrus.WithError(err).Error("scan failed") + c.JSON(http.StatusInternalServerError, gin.H{"error": "scan failed"}) + return + } + + c.JSON(http.StatusOK, accounts) +} diff --git a/internal/api/v1/endpoints.go b/internal/api/v1/endpoints.go index 74c5239df..edd201c96 100644 --- a/internal/api/v1/endpoints.go +++ b/internal/api/v1/endpoints.go @@ -56,6 +56,9 @@ func Endpoints(ctx context.Context, telemetry bool, analytics utils.Analytics, d router.POST("/alerts/test", api.TestEndpointHandler) router.GET("/telemetry", api.TelemetryHandler) + router.GET("/is_onboarded", api.IsOnboardedHandler) + + router.GET("/cloud_accounts", api.ListCloudAccountsHandler) router.NoRoute(gin.WrapH(http.FileServer(assetFS()))) diff --git a/internal/config/load.go b/internal/config/load.go index e0183cb89..feaee4d56 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/BurntSushi/toml" @@ -19,6 +20,7 @@ import ( "github.com/mongodb-forks/digest" "github.com/oracle/oci-go-sdk/common" "github.com/scaleway/scaleway-sdk-go/scw" + "github.com/tailwarden/komiser/models" . "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/providers" "github.com/tailwarden/komiser/utils" @@ -26,6 +28,7 @@ import ( "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/regions" tccvm "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312" + "github.com/uptrace/bun" "go.mongodb.org/atlas/mongodbatlas" "golang.org/x/oauth2" "golang.org/x/oauth2/google" @@ -62,27 +65,39 @@ func loadConfigFromBytes(b []byte) (*Config, error) { return &config, nil } -func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config, []providers.ProviderClient, error) { +func Load(configPath string, telemetry bool, analytics utils.Analytics, db *bun.DB) (*Config, []providers.ProviderClient, []models.Account, error) { config, err := loadConfigFromFile(configPath) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if len(config.SQLite.File) == 0 && config.Postgres.URI == "" { - return nil, nil, errors.New("postgres URI or sqlite file is missing") + return nil, nil, nil, errors.New("postgres URI or sqlite file is missing") } clients := make([]providers.ProviderClient, 0) + accounts := make([]models.Account, 0) if len(config.AWS) > 0 { for _, account := range config.AWS { + cloudAccount := models.Account{ + Provider: "AWS", + Name: account.Name, + Credentials: map[string]string{ + "profile": account.Profile, + "path": account.Path, + "source": account.Source, + }, + } + accounts = append(accounts, cloudAccount) + if account.Source == "CREDENTIALS_FILE" { if len(account.Path) > 0 { cfg, err := awsConfig.LoadDefaultConfig(context.Background(), awsConfig.WithSharedConfigProfile(account.Profile), awsConfig.WithSharedCredentialsFiles( []string{account.Path}, )) if err != nil { - return nil, nil, err + return nil, nil, nil, err } clients = append(clients, providers.ProviderClient{ AWSClient: &cfg, @@ -91,7 +106,7 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config } else { cfg, err := awsConfig.LoadDefaultConfig(context.Background(), awsConfig.WithSharedConfigProfile(account.Profile)) if err != nil { - return nil, nil, err + return nil, nil, nil, err } clients = append(clients, providers.ProviderClient{ AWSClient: &cfg, @@ -119,6 +134,15 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.DigitalOcean) > 0 { for _, account := range config.DigitalOcean { + cloudAccount := models.Account{ + Provider: "DigitalOcean", + Name: account.Name, + Credentials: map[string]string{ + "token": account.Token, + }, + } + accounts = append(accounts, cloudAccount) + client := godo.NewFromToken(account.Token) clients = append(clients, providers.ProviderClient{ DigitalOceanClient: client, @@ -135,6 +159,16 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.Oci) > 0 { for _, account := range config.Oci { + cloudAccount := models.Account{ + Provider: "OCI", + Name: account.Name, + Credentials: map[string]string{ + "profile": account.Profile, + "source": account.Source, + }, + } + accounts = append(accounts, cloudAccount) + if account.Source == "CREDENTIALS_FILE" { client := common.DefaultConfigProvider() clients = append(clients, providers.ProviderClient{ @@ -153,6 +187,15 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.Civo) > 0 { for _, account := range config.Civo { + cloudAccount := models.Account{ + Provider: "Civo", + Name: account.Name, + Credentials: map[string]string{ + "token": account.Token, + }, + } + accounts = append(accounts, cloudAccount) + client, err := civogo.NewClient(account.Token, "LON1") if err != nil { log.Fatal(err) @@ -172,6 +215,17 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.Kubernetes) > 0 { for _, account := range config.Kubernetes { + cloudAccount := models.Account{ + Provider: "Kubernetes", + Name: account.Name, + Credentials: map[string]string{ + "path": account.Path, + "contexts": strings.Join(account.Contexts, ";"), + }, + } + + accounts = append(accounts, cloudAccount) + kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( &clientcmd.ClientConfigLoadingRules{ExplicitPath: account.Path}, &clientcmd.ConfigOverrides{}).ClientConfig() @@ -199,6 +253,16 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.Linode) > 0 { for _, account := range config.Linode { + cloudAccount := models.Account{ + Provider: "Linode", + Name: account.Name, + Credentials: map[string]string{ + "token": account.Token, + }, + } + + accounts = append(accounts, cloudAccount) + tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: account.Token}) oauth2Client := &http.Client{ Transport: &oauth2.Transport{ @@ -222,6 +286,17 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.Tencent) > 0 { for _, account := range config.Tencent { + cloudAccount := models.Account{ + Provider: "Tencent", + Name: account.Name, + Credentials: map[string]string{ + "secretId": account.SecretID, + "secretKey": account.SecretKey, + }, + } + + accounts = append(accounts, cloudAccount) + credential := tccommon.NewCredential(account.SecretID, account.SecretKey) cpf := profile.NewClientProfile() cpf.Language = "en-US" @@ -245,6 +320,19 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.Azure) > 0 { for _, account := range config.Azure { + cloudAccount := models.Account{ + Provider: "Azure", + Name: account.Name, + Credentials: map[string]string{ + "clientId": account.ClientId, + "clientSecret": account.ClientSecret, + "tenantId": account.TenantId, + "subscriptionId": account.SubscriptionId, + }, + } + + accounts = append(accounts, cloudAccount) + creds, err := azidentity.NewClientSecretCredential(account.TenantId, account.ClientId, account.ClientSecret, &azidentity.ClientSecretCredentialOptions{}) if err != nil { log.Fatal(err) @@ -270,6 +358,18 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.Scaleway) > 0 { for _, account := range config.Scaleway { + cloudAccount := models.Account{ + Provider: "Scaleway", + Name: account.Name, + Credentials: map[string]string{ + "accessKey": account.AccessKey, + "secretKey": account.SecretKey, + "organizationId": account.OrganizationId, + }, + } + + accounts = append(accounts, cloudAccount) + client, err := scw.NewClient( scw.WithDefaultOrganizationID(account.OrganizationId), scw.WithAuth(account.AccessKey, account.SecretKey), @@ -293,6 +393,18 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config if len(config.MongoDBAtlas) > 0 { for _, account := range config.MongoDBAtlas { + cloudAccount := models.Account{ + Provider: "MongoDB", + Name: account.Name, + Credentials: map[string]string{ + "publicKey": account.PublicApiKey, + "privateKey": account.PrivateApiKey, + "organizationId": account.OrganizationID, + }, + } + + accounts = append(accounts, cloudAccount) + t := digest.NewTransport(account.PublicApiKey, account.PrivateApiKey) tc, err := t.Client() if err != nil { @@ -308,8 +420,17 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config } if len(config.GCP) > 0 { - // Initialize a GCP client for _, account := range config.GCP { + cloudAccount := models.Account{ + Provider: "GCP", + Name: account.Name, + Credentials: map[string]string{ + "accountKey": account.ServiceAccountKeyPath, + }, + } + + accounts = append(accounts, cloudAccount) + data, err := ioutil.ReadFile(account.ServiceAccountKeyPath) if err != nil { log.Fatal(err) @@ -329,5 +450,5 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*Config } } - return config, clients, nil + return config, clients, accounts, nil } diff --git a/internal/internal.go b/internal/internal.go index 236cd8ba2..eb0c7240f 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -61,12 +61,12 @@ func Exec(address string, port int, configPath string, telemetry bool, a utils.A ctx := context.Background() - cfg, clients, err := config.Load(configPath, telemetry, analytics) + cfg, clients, accounts, err := config.Load(configPath, telemetry, analytics, db) if err != nil { return err } - err = setupSchema(cfg) + err = setupSchema(cfg, accounts) if err != nil { return err } @@ -179,7 +179,7 @@ func runServer(address string, port int, telemetry bool, cfg models.Config) erro return nil } -func setupSchema(c *models.Config) error { +func setupSchema(c *models.Config, accounts []models.Account) error { var sqldb *sql.DB var err error @@ -216,6 +216,18 @@ func setupSchema(c *models.Config) error { return err } + _, err = db.NewCreateTable().Model((*models.Account)(nil)).IfNotExists().Exec(context.Background()) + if err != nil { + return err + } + + for _, account := range accounts { + _, err = db.NewInsert().Model(&account).Exec(context.Background()) + if err != nil { + log.Warnf("%s account cannot be inserted to database", account.Provider) + } + } + // Created pre-defined views untaggedResourcesView := models.View{ Name: "Untagged resources", diff --git a/models/account.go b/models/account.go new file mode 100644 index 000000000..4a05c7d50 --- /dev/null +++ b/models/account.go @@ -0,0 +1,8 @@ +package models + +type Account struct { + Id int64 `json:"id" bun:"id,pk,autoincrement"` + Provider string `json:"provider"` + Name string `json:"name"` + Credentials map[string]string `json:"credentials" bun:"credentials,unique"` +} From 70af728370d3c66d2756783105a26da5164d08c8 Mon Sep 17 00:00:00 2001 From: Mohamed Labouardy Date: Wed, 20 Sep 2023 09:12:40 +0300 Subject: [PATCH 2/5] feat: expose new cloud account endpoint --- .../onboarding-wizard/DatabasePurplin.tsx | 2 +- .../onboarding-wizard/PurplinCloud.tsx | 2 +- .../onboarding-wizard/SelectInput.tsx | 10 +++---- dashboard/components/select/Select.tsx | 2 +- dashboard/pages/cloud-accounts.tsx | 4 +-- dashboard/pages/onboarding/choose-cloud.tsx | 2 +- handlers/accounts_handler.go | 30 +++++++++++++++++++ internal/api/v1/endpoints.go | 1 + 8 files changed, 42 insertions(+), 11 deletions(-) diff --git a/dashboard/components/onboarding-wizard/DatabasePurplin.tsx b/dashboard/components/onboarding-wizard/DatabasePurplin.tsx index 55f6ade5f..f81391554 100644 --- a/dashboard/components/onboarding-wizard/DatabasePurplin.tsx +++ b/dashboard/components/onboarding-wizard/DatabasePurplin.tsx @@ -7,7 +7,7 @@ interface DatabasePurplinProps { function DatabasePurplin({ database }: DatabasePurplinProps) { return (
-
+
-
+
{`${provider}
{icon}
@@ -49,7 +49,7 @@ function SelectInput({ { 'outline-2 outline-primary': isOpen } )} > -
+
{displayValues[index].icon && displayValues[index].icon} {displayValues[index].label}
@@ -61,7 +61,7 @@ function SelectInput({ onClick={toggle} className="fixed inset-0 z-20 hidden animate-fade-in bg-transparent opacity-0 sm:block" >
-
+
{values.map((item, idx) => { const isActive = value === item; @@ -69,7 +69,7 @@ function SelectInput({
-
+
{values.map((item, idx) => { const isActive = value === item; diff --git a/dashboard/pages/cloud-accounts.tsx b/dashboard/pages/cloud-accounts.tsx index a984703be..acbe5fd4e 100644 --- a/dashboard/pages/cloud-accounts.tsx +++ b/dashboard/pages/cloud-accounts.tsx @@ -51,7 +51,7 @@ function CloudAccounts() {
{status.state} -
+
{status.message}
diff --git a/dashboard/pages/onboarding/choose-cloud.tsx b/dashboard/pages/onboarding/choose-cloud.tsx index 07f2992da..c57b9217e 100644 --- a/dashboard/pages/onboarding/choose-cloud.tsx +++ b/dashboard/pages/onboarding/choose-cloud.tsx @@ -92,7 +92,7 @@ export default function ChooseCloud() { width={500} height={120} /> -
+
{`${provider} Date: Wed, 20 Sep 2023 09:14:19 +0300 Subject: [PATCH 3/5] feat: expose delete cloud account endpoint --- handlers/accounts_handler.go | 13 +++++++++++++ internal/api/v1/endpoints.go | 1 + 2 files changed, 14 insertions(+) diff --git a/handlers/accounts_handler.go b/handlers/accounts_handler.go index d68a3370b..2d9ece821 100644 --- a/handlers/accounts_handler.go +++ b/handlers/accounts_handler.go @@ -71,3 +71,16 @@ func (handler *ApiHandler) NewCloudAccountHandler(c *gin.Context) { c.JSON(http.StatusCreated, account) } + +func (handler *ApiHandler) DeleteCloudAccountHandler(c *gin.Context) { + accountId := c.Param("id") + + account := new(models.Account) + _, err := handler.db.NewDelete().Model(account).Where("id = ?", accountId).Exec(handler.ctx) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"message": "account has been deleted"}) +} diff --git a/internal/api/v1/endpoints.go b/internal/api/v1/endpoints.go index e878c5b26..dde45a700 100644 --- a/internal/api/v1/endpoints.go +++ b/internal/api/v1/endpoints.go @@ -60,6 +60,7 @@ func Endpoints(ctx context.Context, telemetry bool, analytics utils.Analytics, d router.GET("/cloud_accounts", api.ListCloudAccountsHandler) router.POST("/cloud_accounts", api.NewCloudAccountHandler) + router.DELETE("/cloud_accounts/:id", api.DeleteCloudAccountHandler) router.NoRoute(gin.WrapH(http.FileServer(assetFS()))) From db94739983ecac1e4721ef6311aaf717a92ec129 Mon Sep 17 00:00:00 2001 From: Mohamed Labouardy Date: Wed, 20 Sep 2023 09:16:17 +0300 Subject: [PATCH 4/5] feat: expose update cloud account endpoint --- handlers/accounts_handler.go | 19 +++++++++++++++++++ internal/api/v1/endpoints.go | 1 + 2 files changed, 20 insertions(+) diff --git a/handlers/accounts_handler.go b/handlers/accounts_handler.go index 2d9ece821..936b842ab 100644 --- a/handlers/accounts_handler.go +++ b/handlers/accounts_handler.go @@ -84,3 +84,22 @@ func (handler *ApiHandler) DeleteCloudAccountHandler(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"message": "account has been deleted"}) } + +func (handler *ApiHandler) UpdateCloudAccountHandler(c *gin.Context) { + accountId := c.Param("id") + + var account models.Account + err := json.NewDecoder(c.Request.Body).Decode(&account) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + _, err = handler.db.NewUpdate().Model(&account).Column("name", "provider", "credentials").Where("id = ?", accountId).Exec(handler.ctx) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, alert) +} diff --git a/internal/api/v1/endpoints.go b/internal/api/v1/endpoints.go index dde45a700..f29f4b106 100644 --- a/internal/api/v1/endpoints.go +++ b/internal/api/v1/endpoints.go @@ -61,6 +61,7 @@ func Endpoints(ctx context.Context, telemetry bool, analytics utils.Analytics, d router.GET("/cloud_accounts", api.ListCloudAccountsHandler) router.POST("/cloud_accounts", api.NewCloudAccountHandler) router.DELETE("/cloud_accounts/:id", api.DeleteCloudAccountHandler) + router.PUT("/cloud_accounts/:id", api.UpdateCloudAccountHandler) router.NoRoute(gin.WrapH(http.FileServer(assetFS()))) From 6c2461ef891fb04ef806e460d5322707095742c6 Mon Sep 17 00:00:00 2001 From: Mohamed Labouardy Date: Wed, 20 Sep 2023 09:17:30 +0300 Subject: [PATCH 5/5] fix: remove debug logs --- cmd/start.go | 2 -- handlers/accounts_handler.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 4f3f2a57a..830d6ed75 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -2,7 +2,6 @@ package cmd import ( "errors" - "fmt" "time" "github.com/getsentry/sentry-go" @@ -62,7 +61,6 @@ var startCmd = &cobra.Command{ err = internal.Exec(address, port, file, telemetry, analytics, regions, cmd) if err != nil { - fmt.Println(err) return err } diff --git a/handlers/accounts_handler.go b/handlers/accounts_handler.go index 936b842ab..a046468c4 100644 --- a/handlers/accounts_handler.go +++ b/handlers/accounts_handler.go @@ -101,5 +101,5 @@ func (handler *ApiHandler) UpdateCloudAccountHandler(c *gin.Context) { return } - c.JSON(http.StatusOK, alert) + c.JSON(http.StatusOK, account) }