Send logs to the default URL when Cluster is in deletion state

gardener · May 28, 2021 · 54f0fb7 · 54f0fb7
1 parent c4a3498
commit 54f0fb7
Show file tree

Hide file tree

Showing 464 changed files with 28,146 additions and 4,916 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -13,6 +13,8 @@ COPY --from=builder /go/src/github.com/gardener/logging/build /source/plugins
 
 WORKDIR /
 
+ENTRYPOINT ["cp","/source/plugins/.","/plugins", "-fr"]
+
 #############      curator       #############
 FROM eu.gcr.io/gardener-project/3rd/alpine:3.12.3 AS curator
 

diff --git a/README.md b/README.md
@@ -44,7 +44,9 @@ It also adds additional configurations that aim to improve plugin's performance
 | DropLogEntryWithoutK8sMetadata | When metadata is missing for the log entry, it will be dropped | `false`
 | ControllerSyncTimeout | Time to wait for cluster object synchronization | 60 seconds
 | NumberOfBatchIDs | The number of id per batch. This increase the number of loki label streams | 10
-
+| SendDeletedClustersLogsToDefaultClient | When cluster is marked for deletion the logs will be send to the default url `URL` | `false`
+| DeletedClientTimeExpiration | The time duration after a client for deleted cluster will be considered for expired | 1 hour
+| CleanExpiredClientsPeriod | Clean the expired clients every `CleanExpiredClientsPeriod` | 24 hours
 ### Labels
 
 Labels are used to [query logs](https://github.com/grafana/loki/blob/v1.5.0/docs/logql.md) `{container_name="nginx", cluster="us-west1"}`, they are usually metadata about the workload producing the log stream (`instance`, `container_name`, `region`, `cluster`, `level`).  In Loki labels are indexed consequently you should be cautious when choosing them (high cardinality label values can have performance drastic impact).

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-v0.35.0-dev
+v0.35-dev
diff --git a/cmd/fluent-bit-loki-plugin/out_loki.go b/cmd/fluent-bit-loki-plugin/out_loki.go
@@ -56,6 +56,7 @@ func init() {
 	var logLevel logging.Level
 	_ = logLevel.Set("info")
 	logger = log.With(newLogger(logLevel), "ts", log.DefaultTimestampUTC, "caller", "main")
+
 	kubernetesCleint, err := getInclusterKubernetsClient()
 	if err != nil {
 		panic(err)
@@ -96,39 +97,42 @@ func FLBPluginInit(ctx unsafe.Pointer) int {
 
 	level.Info(logger).Log("[flb-go]", "Starting fluent-bit-go-loki", "version", version.Info())
 	paramLogger := log.With(logger, "[flb-go]", "provided parameter")
-	level.Info(paramLogger).Log("URL", conf.ClientConfig.URL)
-	level.Info(paramLogger).Log("TenantID", conf.ClientConfig.TenantID)
-	level.Info(paramLogger).Log("BatchWait", conf.ClientConfig.BatchWait)
-	level.Info(paramLogger).Log("BatchSize", conf.ClientConfig.BatchSize)
-	level.Info(paramLogger).Log("Labels", conf.ClientConfig.ExternalLabels)
+	level.Info(paramLogger).Log("URL", conf.ClientConfig.GrafanaLokiConfig.URL)
+	level.Info(paramLogger).Log("TenantID", conf.ClientConfig.GrafanaLokiConfig.TenantID)
+	level.Info(paramLogger).Log("BatchWait", conf.ClientConfig.GrafanaLokiConfig.BatchWait)
+	level.Info(paramLogger).Log("BatchSize", conf.ClientConfig.GrafanaLokiConfig.BatchSize)
+	level.Info(paramLogger).Log("Labels", conf.ClientConfig.GrafanaLokiConfig.ExternalLabels)
 	level.Info(paramLogger).Log("LogLevel", conf.LogLevel.String())
-	level.Info(paramLogger).Log("AutoKubernetesLabels", conf.AutoKubernetesLabels)
-	level.Info(paramLogger).Log("RemoveKeys", fmt.Sprintf("%+v", conf.RemoveKeys))
-	level.Info(paramLogger).Log("LabelKeys", fmt.Sprintf("%+v", conf.LabelKeys))
-	level.Info(paramLogger).Log("LineFormat", conf.LineFormat)
-	level.Info(paramLogger).Log("DropSingleKey", conf.DropSingleKey)
-	level.Info(paramLogger).Log("LabelMapPath", fmt.Sprintf("%+v", conf.LabelMap))
-	level.Info(paramLogger).Log("SortByTimestamp", fmt.Sprintf("%+v", conf.SortByTimestamp))
-	level.Info(paramLogger).Log("DynamicHostPath", fmt.Sprintf("%+v", conf.DynamicHostPath))
-	level.Info(paramLogger).Log("DynamicHostPrefix", fmt.Sprintf("%+v", conf.DynamicHostPrefix))
-	level.Info(paramLogger).Log("DynamicHostSuffix", fmt.Sprintf("%+v", conf.DynamicHostSuffix))
-	level.Info(paramLogger).Log("DynamicHostRegex", fmt.Sprintf("%+v", conf.DynamicHostRegex))
-	level.Info(paramLogger).Log("Timeout", fmt.Sprintf("%+v", conf.ClientConfig.Timeout))
-	level.Info(paramLogger).Log("MinBackoff", fmt.Sprintf("%+v", conf.ClientConfig.BackoffConfig.MinBackoff))
-	level.Info(paramLogger).Log("MaxBackoff", fmt.Sprintf("%+v", conf.ClientConfig.BackoffConfig.MaxBackoff))
-	level.Info(paramLogger).Log("MaxRetries", fmt.Sprintf("%+v", conf.ClientConfig.BackoffConfig.MaxRetries))
-	level.Info(paramLogger).Log("Buffer", fmt.Sprintf("%+v", conf.BufferConfig.Buffer))
-	level.Info(paramLogger).Log("BufferType", fmt.Sprintf("%+v", conf.BufferConfig.BufferType))
-	level.Info(paramLogger).Log("QueueDir", fmt.Sprintf("%+v", conf.BufferConfig.DqueConfig.QueueDir))
-	level.Info(paramLogger).Log("QueueSegmentSize", fmt.Sprintf("%+v", conf.BufferConfig.DqueConfig.QueueSegmentSize))
-	level.Info(paramLogger).Log("QueueSync", fmt.Sprintf("%+v", conf.BufferConfig.DqueConfig.QueueSync))
-	level.Info(paramLogger).Log("QueueName", fmt.Sprintf("%+v", conf.BufferConfig.DqueConfig.QueueName))
-	level.Info(paramLogger).Log("FallbackToTagWhenMetadataIsMissing", fmt.Sprintf("%+v", conf.KubernetesMetadata.FallbackToTagWhenMetadataIsMissing))
-	level.Info(paramLogger).Log("TagKey", fmt.Sprintf("%+v", conf.KubernetesMetadata.TagKey))
-	level.Info(paramLogger).Log("TagPrefix", fmt.Sprintf("%+v", conf.KubernetesMetadata.TagPrefix))
-	level.Info(paramLogger).Log("TagExpression", fmt.Sprintf("%+v", conf.KubernetesMetadata.TagExpression))
-	level.Info(paramLogger).Log("DropLogEntryWithoutK8sMetadata", fmt.Sprintf("%+v", conf.KubernetesMetadata.DropLogEntryWithoutK8sMetadata))
-	level.Info(paramLogger).Log("NumberOfBatchIDs", fmt.Sprintf("%+v", conf.NumberOfBatchIDs))
+	level.Info(paramLogger).Log("AutoKubernetesLabels", conf.PluginConfig.AutoKubernetesLabels)
+	level.Info(paramLogger).Log("RemoveKeys", fmt.Sprintf("%+v", conf.PluginConfig.RemoveKeys))
+	level.Info(paramLogger).Log("LabelKeys", fmt.Sprintf("%+v", conf.PluginConfig.LabelKeys))
+	level.Info(paramLogger).Log("LineFormat", conf.PluginConfig.LineFormat)
+	level.Info(paramLogger).Log("DropSingleKey", conf.PluginConfig.DropSingleKey)
+	level.Info(paramLogger).Log("LabelMapPath", fmt.Sprintf("%+v", conf.PluginConfig.LabelMap))
+	level.Info(paramLogger).Log("SortByTimestamp", fmt.Sprintf("%+v", conf.ClientConfig.SortByTimestamp))
+	level.Info(paramLogger).Log("DynamicHostPath", fmt.Sprintf("%+v", conf.PluginConfig.DynamicHostPath))
+	level.Info(paramLogger).Log("DynamicHostPrefix", fmt.Sprintf("%+v", conf.ControllerConfig.DynamicHostPrefix))
+	level.Info(paramLogger).Log("DynamicHostSuffix", fmt.Sprintf("%+v", conf.ControllerConfig.DynamicHostSuffix))
+	level.Info(paramLogger).Log("DynamicHostRegex", fmt.Sprintf("%+v", conf.PluginConfig.DynamicHostRegex))
+	level.Info(paramLogger).Log("Timeout", fmt.Sprintf("%+v", conf.ClientConfig.GrafanaLokiConfig.Timeout))
+	level.Info(paramLogger).Log("MinBackoff", fmt.Sprintf("%+v", conf.ClientConfig.GrafanaLokiConfig.BackoffConfig.MinBackoff))
+	level.Info(paramLogger).Log("MaxBackoff", fmt.Sprintf("%+v", conf.ClientConfig.GrafanaLokiConfig.BackoffConfig.MaxBackoff))
+	level.Info(paramLogger).Log("MaxRetries", fmt.Sprintf("%+v", conf.ClientConfig.GrafanaLokiConfig.BackoffConfig.MaxRetries))
+	level.Info(paramLogger).Log("Buffer", fmt.Sprintf("%+v", conf.ClientConfig.BufferConfig.Buffer))
+	level.Info(paramLogger).Log("BufferType", fmt.Sprintf("%+v", conf.ClientConfig.BufferConfig.BufferType))
+	level.Info(paramLogger).Log("QueueDir", fmt.Sprintf("%+v", conf.ClientConfig.BufferConfig.DqueConfig.QueueDir))
+	level.Info(paramLogger).Log("QueueSegmentSize", fmt.Sprintf("%+v", conf.ClientConfig.BufferConfig.DqueConfig.QueueSegmentSize))
+	level.Info(paramLogger).Log("QueueSync", fmt.Sprintf("%+v", conf.ClientConfig.BufferConfig.DqueConfig.QueueSync))
+	level.Info(paramLogger).Log("QueueName", fmt.Sprintf("%+v", conf.ClientConfig.BufferConfig.DqueConfig.QueueName))
+	level.Info(paramLogger).Log("FallbackToTagWhenMetadataIsMissing", fmt.Sprintf("%+v", conf.PluginConfig.KubernetesMetadata.FallbackToTagWhenMetadataIsMissing))
+	level.Info(paramLogger).Log("TagKey", fmt.Sprintf("%+v", conf.PluginConfig.KubernetesMetadata.TagKey))
+	level.Info(paramLogger).Log("TagPrefix", fmt.Sprintf("%+v", conf.PluginConfig.KubernetesMetadata.TagPrefix))
+	level.Info(paramLogger).Log("TagExpression", fmt.Sprintf("%+v", conf.PluginConfig.KubernetesMetadata.TagExpression))
+	level.Info(paramLogger).Log("DropLogEntryWithoutK8sMetadata", fmt.Sprintf("%+v", conf.PluginConfig.KubernetesMetadata.DropLogEntryWithoutK8sMetadata))
+	level.Info(paramLogger).Log("NumberOfBatchIDs", fmt.Sprintf("%+v", conf.ClientConfig.NumberOfBatchIDs))
+	level.Info(paramLogger).Log("SendDeletedClustersLogsToDefaultClient", fmt.Sprintf("%+v", conf.ControllerConfig.SendDeletedClustersLogsToDefaultClient))
+	level.Info(paramLogger).Log("DeletedClientTimeExpiration", fmt.Sprintf("%+v", conf.ControllerConfig.DeletedClientTimeExpiration))
+	level.Info(paramLogger).Log("CleanExpiredClientsPeriod", fmt.Sprintf("%+v", conf.ControllerConfig.CleanExpiredClientsPeriod))
 
 	plugin, err := lokiplugin.NewPlugin(informer, conf, logger)
 	if err != nil {
@@ -216,8 +220,6 @@ func getInclusterKubernetsClient() (versioned.Interface, error) {
 	if err != nil {
 		return nil, err
 	}
-	//fake.NewSimpleClientset().
-	//return kubernetes.NewForConfig(config)
 	return gardenerclientsetversioned.NewForConfig(config)
 }
 

diff --git a/go.mod b/go.mod
@@ -11,16 +11,24 @@ require (
 	github.com/go-logfmt/logfmt v0.5.0
 	github.com/gobuffalo/packr/v2 v2.8.0
 	github.com/golang/mock v1.4.3
+	github.com/google/go-cmp v0.5.1 // indirect
 	github.com/grafana/loki v1.6.0
 	github.com/joncrlsn/dque v2.2.1-0.20200515025108-956d14155fa2+incompatible
 	github.com/json-iterator/go v1.1.10
+	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.8.1
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.8.0
 	github.com/prometheus/common v0.14.0
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/cobra v1.1.3 // indirect
 	github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9
-	gopkg.in/yaml.v2 v2.3.0
+	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/apimachinery v0.18.6
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/code-generator v0.18.3