Skip to content

Commit

Permalink
Merge branch 'develop' into dependabot/npm_and_yarn/dashboard/commitl…
Browse files Browse the repository at this point in the history
…int/cli-19.4.0
  • Loading branch information
Azanul authored Sep 2, 2024
2 parents 7a0aad0 + 6613a76 commit 7799d12
Show file tree
Hide file tree
Showing 13 changed files with 672 additions and 596 deletions.
7 changes: 4 additions & 3 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ jobs:
go-version-file: go.mod

- name: golangci-lint
uses: golangci/golangci-lint-action@v3
uses: golangci/golangci-lint-action@v6
with:
cache-invalidation-interval: 15
args: --timeout=10m
version: v1.54.2
skip-pkg-cache: true
version: v1.59
skip-cache: true
1,004 changes: 423 additions & 581 deletions dashboard/package-lock.json

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions dashboard/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
},
"dependencies": {
"@formbricks/js": "^2.1.1",
"@sentry/react": "^8.25.0",
"@sentry/react": "^8.26.0",
"@sentry/tracing": "^7.41.0",
"@tippyjs/react": "^4.2.6",
"chart.js": "^4.2.1",
Expand All @@ -44,8 +44,8 @@
"@commitlint/config-conventional": "^19.2.2",
"@jest/globals": "^29.5.0",
"@storybook/addon-essentials": "^8.0.5",
"@storybook/addon-interactions": "^8.0.9",
"@storybook/addon-links": "^8.2.4",
"@storybook/addon-interactions": "^8.2.9",
"@storybook/addon-links": "^8.2.9",
"@storybook/addons": "^7.6.17",
"@storybook/blocks": "^8.2.7",
"@storybook/nextjs": "^8.2.5",
Expand All @@ -58,7 +58,7 @@
"@types/cytoscape": "^3.19.16",
"@types/cytoscape-popper": "^2.0.3",
"@types/jest": "^29.5.12",
"@types/node": "20.14.11",
"@types/node": "22.4.1",
"@types/react": "18.3.3",
"@types/react-cytoscapejs": "^1.2.5",
"@types/react-dom": "18.3.0",
Expand All @@ -68,7 +68,7 @@
"eslint-config-airbnb-base": "^15.0.0",
"eslint-config-next": "14.2.3",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-jest": "^27.6.0",
"eslint-plugin-jest": "^28.8.0",
"eslint-plugin-prettier": "^5.2.1",
"eslint-plugin-storybook": "^0.8.0",
"husky": "^9.0.11",
Expand All @@ -78,6 +78,6 @@
"prettier": "^3.2.5",
"prettier-plugin-tailwindcss": "^0.5.12",
"storybook": "^8.1.10",
"tailwindcss": "^3.4.4"
"tailwindcss": "^3.4.10"
}
}
10 changes: 5 additions & 5 deletions docs/configuration/cloud-providers/ovh.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ You will be issued three keys: the application key called AK, tour application s
endpoint="ovh-eu"
application_key="my_app_key"
application_secret="my_application_secret"
my_consumer_key="my_consumer_key"
consumer_key="my_consumer_key"
[sqlite]
file="komiser.db
Expand All @@ -90,28 +90,28 @@ Simply add more authentication blocks to the configuration file
endpoint="ovh-eu"
application_key="my_app_key"
application_secret="my_application_secret"
my_consumer_key="my_consumer_key"
consumer_key="my_consumer_key"
[[ovh]]
name="ovh-account-1"
endpoint="ovh-eu"
application_key="my_app_key"
application_secret="my_application_secret"
my_consumer_key="my_consumer_key"
consumer_key="my_consumer_key"
[[ovh]]
name="ovh-account-2"
endpoint="ovh-eu"
application_key="my_app_key"
application_secret="my_application_secret"
my_consumer_key="my_consumer_key"
consumer_key="my_consumer_key"
[[ovh]]
name="ovh-account-3"
endpoint="ovh-eu"
application_key="my_app_key"
application_secret="my_application_secret"
my_consumer_key="my_consumer_key"
consumer_key="my_consumer_key"
[sqlite]
file="komiser.db
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,8 @@ github.com/oracle/oci-go-sdk v24.3.0+incompatible h1:x4mcfb4agelf1O4/1/auGlZ1lr9
github.com/oracle/oci-go-sdk v24.3.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888=
github.com/ovh/go-ovh v1.4.3 h1:Gs3V823zwTFpzgGLZNI6ILS4rmxZgJwJCz54Er9LwD0=
github.com/ovh/go-ovh v1.4.3/go.mod h1:AkPXVtgwB6xlKblMjRKJJmjRp+ogrE7fz2lVgcQY8SY=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
Expand Down
8 changes: 8 additions & 0 deletions handlers/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"time"

"github.com/BurntSushi/toml"
"github.com/patrickmn/go-cache"
tccommon "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
tccvm "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312"

Expand Down Expand Up @@ -50,6 +51,11 @@ import (
"github.com/tailwarden/komiser/utils"
)

const (
CACHE_DURATION = 3
CLEANUP_DURATION = 4
)

var mu sync.Mutex

func triggerFetchingWorkflow(ctx context.Context, client providers.ProviderClient, provider string, db *bun.DB, regions []string, wp *providers.WorkerPool) {
Expand Down Expand Up @@ -247,8 +253,10 @@ func makeClientFromAccount(account models.Account) (*providers.ProviderClient, e
OpencostBaseUrl: account.Credentials["opencostBaseUrl"],
}

cache := cache.New(CACHE_DURATION, CLEANUP_DURATION)
return &providers.ProviderClient{
K8sClient: &client,
Cache: cache, // Alpha feature for dependency
Name: account.Name,
}, nil
}
Expand Down
4 changes: 4 additions & 0 deletions internal/config/load.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ import (
"github.com/mongodb-forks/digest"
"github.com/oracle/oci-go-sdk/common"
"github.com/ovh/go-ovh/ovh"
"github.com/patrickmn/go-cache"
"github.com/scaleway/scaleway-sdk-go/scw"
"github.com/tailwarden/komiser/handlers"
"github.com/tailwarden/komiser/models"
"github.com/tailwarden/komiser/providers"
"github.com/tailwarden/komiser/utils"
Expand Down Expand Up @@ -241,8 +243,10 @@ func Load(configPath string, telemetry bool, analytics utils.Analytics) (*models
OpencostBaseUrl: account.OpencostBaseUrl,
}

cache := cache.New(handlers.CACHE_DURATION, handlers.CACHE_DURATION)
clients = append(clients, providers.ProviderClient{
K8sClient: &client,
Cache: cache,
Name: account.Name,
})
}
Expand Down
89 changes: 89 additions & 0 deletions providers/k8s/core/pods.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"github.com/tailwarden/komiser/models"
"github.com/tailwarden/komiser/providers"
oc "github.com/tailwarden/komiser/providers/k8s/opencost"
"github.com/tailwarden/komiser/utils"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

Expand Down Expand Up @@ -68,6 +70,7 @@ func Pods(ctx context.Context, client providers.ProviderClient) ([]models.Resour
ResourceId: string(pod.UID),
Name: pod.Name,
Region: pod.Namespace,
Relations: getPodRelation(pod, client),
Cost: cost,
CreatedAt: pod.CreationTimestamp.Time,
FetchedAt: time.Now(),
Expand All @@ -90,3 +93,89 @@ func Pods(ctx context.Context, client providers.ProviderClient) ([]models.Resour
}).Info("Fetched resources")
return resources, nil
}

func getPodRelation(pod v1.Pod, client providers.ProviderClient) []models.Link {

var rel []models.Link
var err error
owners := pod.GetOwnerReferences()
for _, owner := range owners {
rel = append(rel, models.Link{
ResourceID: string(owner.UID),
Type: owner.Kind,
Name: owner.Name,
Relation: "USES",
})
}

// check if service was cached before
// if not call the service cache function, get list of services and cahce it for further use
services, ok := client.Cache.Get(utils.SERVICES)
if !ok {
services, err = utils.K8s_Cache(&client, utils.SERVICES)
}

if err == nil {
serviceList, _ := services.(*v1.ServiceList)

for _, service := range serviceList.Items {
selector := service.Spec.Selector
if selectorMatchesPodLabels(selector, pod.Labels) {
rel = append(rel, models.Link{
ResourceID: string(service.UID),
Type: "Service",
Name: service.Name,
Relation: "USES",
})
}
}
}

// for nodes
node, err := client.K8sClient.Client.CoreV1().Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{})
if err == nil {
rel = append(rel, models.Link{
ResourceID: string(node.UID),
Type: "Node",
Name: node.Name,
Relation: "USES",
})
}

// for namespace
namespace, err := client.K8sClient.Client.CoreV1().Namespaces().Get(context.TODO(), pod.Namespace, metav1.GetOptions{})
if err == nil {
rel = append(rel, models.Link{
ResourceID: string(namespace.UID),
Type: "Namespace",
Name: namespace.Name,
Relation: "USES",
})
}

// for serviceAccount
sa, err := client.K8sClient.Client.CoreV1().ServiceAccounts(namespace.Name).Get(context.TODO(), pod.Spec.ServiceAccountName, metav1.GetOptions{})
if err == nil {
rel = append(rel, models.Link{
ResourceID: string(sa.UID),
Type: "ServiceAccount",
Name: sa.Name,
Relation: "USES",
})
}

return rel
}

func selectorMatchesPodLabels(selector, podLabels map[string]string) bool {
if len(selector) == 0 {
return false
}

for key, value := range selector {
if podValue, exists := podLabels[key]; !exists || podValue != value {
return false
}
}
return true
}
99 changes: 99 additions & 0 deletions providers/k8s/core/replicasets.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package core

import (
"context"
"time"

log "github.com/sirupsen/logrus"

"github.com/tailwarden/komiser/models"
"github.com/tailwarden/komiser/providers"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/api/apps/v1"
)

func Replicasets(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) {
resources := make([]models.Resource, 0)

var config metav1.ListOptions

for {
res, err := client.K8sClient.Client.AppsV1().ReplicaSets("").List(ctx, config)
if err != nil {
return nil, err
}

for _, rs := range res.Items {
tags := make([]models.Tag, 0)

for key, value := range rs.Labels {
tags = append(tags, models.Tag{
Key: key,
Value: value,
})
}

if len(rs.OwnerReferences) > 0 {
// we use the owner kind of first owner only as the owner tag
ownerTags := []models.Tag{
{
Key: "owner_kind",
Value: rs.OwnerReferences[0].Kind,
},
{
Key: "owner_name",
Value: rs.OwnerReferences[0].Name,
},
}
tags = append(tags, ownerTags...)
}


resources = append(resources, models.Resource{
Provider: "Kubernetes",
Account: client.Name,
Service: "Replicaset",
ResourceId: string(rs.UID),
Name: rs.Name,
Region: rs.Namespace,
Relations: getReplicasetRelation(rs),
Cost: 0,
CreatedAt: rs.CreationTimestamp.Time,
FetchedAt: time.Now(),
Tags: tags,
})
}

if res.GetContinue() == "" {
break
}

config.Continue = res.GetContinue()
}

log.WithFields(log.Fields{
"provider": "Kubernetes",
"account": client.Name,
"service": "Replicaset",
"resources": len(resources),
}).Info("Fetched resources")
return resources, nil
}


func getReplicasetRelation(rs v1.ReplicaSet) []models.Link {

var rel []models.Link

owners := rs.GetOwnerReferences()
for _, owner := range owners {
rel = append(rel, models.Link{
ResourceID: string(owner.UID),
Type: owner.Kind,
Name: owner.Name,
Relation: "USES",
})
}

return rel
}
2 changes: 1 addition & 1 deletion providers/k8s/k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func FetchResources(ctx context.Context, client providers.ProviderClient, db *bu
log.Printf("[%s][K8s] %s", client.Name, err)
} else {
for _, resource := range resources {
_, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
_, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost, relations=EXCLUDED.relations").Exec(context.Background())
if err != nil {
logrus.WithError(err).Errorf("db trigger failed")
}
Expand Down
Loading

0 comments on commit 7799d12

Please sign in to comment.