Release/0.7.5 (#66)

* Update Readme

* Further prom metrics and alerts

Dockerfile

@@ -24,6 +24,7 @@ COPY vault/ vault
 COPY db/ db
 COPY utils/ utils
 COPY apis/ apis/
+COPY metrics/ metrics/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -ldflags "-X main.developmentMode=false -X main.gitVersion=${VERSION}" -a -o vals-operator main.go

charts/vals-operator/Chart.yaml

@@ -1,7 +1,16 @@
 ---
 apiVersion: v2
 name: vals-operator
-description: This helm chart installs the Digitalis Vals Operator to manage and sync secrets from supported backends into Kubernetes.
+description: >
+  This helm chart installs the Digitalis Vals Operator to manage and sync secrets from supported backends into Kubernetes.
+
+  ## About Vals-Operator
+
+  Here at [Digitalis](https://digitalis.io) we love [vals](https://github.com/helmfile/vals), it's a tool we use daily to keep secrets stored securely. Inspired by this tool,
+  we have created an operator to manage Kubernetes secrets.
+
+  *vals-operator* syncs secrets from any secrets store supported by [vals](https://github.com/helmfile/vals) into Kubernetes. Also, `vals-operator` supports database secrets
+  as provider by [HashiCorp Vault Secret Engine](https://developer.hashicorp.com/vault/docs/secrets/databases).
 icon: https://digitalis.io/wp-content/uploads/2020/06/cropped-Digitalis-512x512-Blue_Digitalis-512x512-Blue-32x32.png
 kubeVersion: ">= 1.19.0-0"
 type: application
@@ -10,7 +19,7 @@ type: application
 version: 0.7.5
 
 # Latest container tag
-appVersion: "v0.7.5"
+appVersion: v0.7.5
 
 maintainers:
 - email: info@digitalis.io

charts/vals-operator/README.md

@@ -1,18 +1,23 @@
-vals-operator
-=============
+# vals-operator
 
-This helm chart installs the Digitalis Vals Operator to manage and sync secrets from supported backends into Kubernetes.
+![Version: 0.7.5](https://img.shields.io/badge/Version-0.7.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.7.5](https://img.shields.io/badge/AppVersion-v0.7.5-informational?style=flat-square)
 
+This helm chart installs the Digitalis Vals Operator to manage and sync secrets from supported backends into Kubernetes.
 ## About Vals-Operator
+Here at [Digitalis](https://digitalis.io) we love [vals](https://github.com/helmfile/vals), it's a tool we use daily to keep secrets stored securely. Inspired by this tool, we have created an operator to manage Kubernetes secrets.
+*vals-operator* syncs secrets from any secrets store supported by [vals](https://github.com/helmfile/vals) into Kubernetes. Also, `vals-operator` supports database secrets as provider by [HashiCorp Vault Secret Engine](https://developer.hashicorp.com/vault/docs/secrets/databases).
 
-Here at [Digitalis](https://digitalis.io) we love [vals](https://github.com/helmfile/vals), it's a tool we use daily to keep secrets stored securely. Inspired by this tool,
-we have created an operator to manage Kubernetes secrets.
+## Maintainers
 
-*vals-operator* syncs secrets from any secrets store supported by [vals](https://github.com/helmfile/vals) into Kubernetes. Also, `vals-operator` supports database secrets
-as provider by [HashiCorp Vault Secret Engine](https://developer.hashicorp.com/vault/docs/secrets/databases).
+| Name | Email | Url |
+| ---- | ------ | --- |
+| Digitalis.IO | <info@digitalis.io> |  |
 
+## Requirements
 
-## Chart Values
+Kubernetes: `>= 1.19.0-0`
+
+## Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
@@ -45,3 +50,6 @@ as provider by [HashiCorp Vault Secret Engine](https://developer.hashicorp.com/v
 | tolerations | list | `[]` |  |
 | volumeMounts | list | `[]` |  |
 | volumes | list | `[]` |  |
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)

charts/vals-operator/templates/prometheusrules.yaml

@@ -22,7 +22,25 @@ spec:
       {{- end }}
           annotations:
             summary: vals-operator database secret not issued
-            description: "Vals operator has been unable to issue database credentials to {{`{{`}}$labels.secret{{`}}`}} in namespace {{`{{`}}$labels.namespace{{`}}`}}"
+            description: "Vals operator has been unable to issue database credentials for secret {{`{{`}}$labels.secret{{`}}`}} in namespace {{`{{`}}$labels.namespace{{`}}`}}"
+      {{- if .Values.prometheusRules.additionalRuleAnnotations }}
+        {{- with .Values.prometheusRules.additionalRuleAnnotations }}
+          {{- toYaml . | nindent 12 }}
+        {{- end }}
+      {{- end }}
+        - alert: ValsOperatorDbSecretExpired
+          expr: time() > vals_operator_dbsecret_expire_time
+          for: 30m
+          labels:
+            severity: warning
+      {{- if .Values.prometheusRules.additionalRuleLabels }}
+        {{- with .Values.prometheusRules.additionalRuleLabels }}
+          {{- toYaml . | nindent 12 }}
+        {{- end }}
+      {{- end }}
+          annotations:
+            summary: vals-operator database secret expired
+            description: "Vals operator database credentials for secret {{`{{`}}$labels.secret{{`}}`}} in namespace {{`{{`}}$labels.namespace{{`}}`}} expired"
       {{- if .Values.prometheusRules.additionalRuleAnnotations }}
         {{- with .Values.prometheusRules.additionalRuleAnnotations }}
           {{- toYaml . | nindent 12 }}

controllers/common.go

@@ -1,6 +1,19 @@
-package controllers
+/*
+Copyright 2023 Digitalis.IO.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
 
-import "github.com/prometheus/client_golang/prometheus"
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package controllers
 
 const (
 	leaseIdLabel               = "vals-operator.digitalis.io/lease-id"
@@ -15,28 +28,3 @@ const (
 	managedByLabel             = "app.kubernetes.io/managed-by"
 	k8sSecretPrefix            = "ref+k8s://"
 )
-
-var (
-	SecretFailures = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "vals_operator_secret_failures",
-			Help: "Number of errors generating secrets",
-		},
-	)
-	DbSecretFailures = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Name: "vals_operator_dbsecret_failures",
-			Help: "Number of errors generating DB secrets",
-		},
-	)
-	SecretError = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "vals_operator_secret_error",
-			Help: "Reports timestamp from when a secret last failed to be updated",
-		}, []string{"secret", "namespace"})
-	DbSecretError = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "vals_operator_dbsecret_error",
-			Help: "Reports timestamp from when a DB secret last failed to be updated",
-		}, []string{"secret", "namespace"})
-)

controllers/dbsecret_controller.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Digitalis.IO.
+Copyright 2023 Digitalis.IO.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -41,6 +41,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	digitalisiov1beta1 "digitalis.io/vals-operator/apis/digitalis.io/v1beta1"
+	dmetrics "digitalis.io/vals-operator/metrics"
 	"digitalis.io/vals-operator/utils"
 	"digitalis.io/vals-operator/vault"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -125,6 +126,9 @@ func (r *DbSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 			if err := r.Update(context.Background(), &dbSecret); err != nil {
 				return ctrl.Result{}, err
 			}
+			/* mark as deleted in prom */
+			dmetrics.DbSecretExpireTime.WithLabelValues(dbSecret.Name, dbSecret.Namespace).Set(0)
+			dmetrics.DbSecretInfo.WithLabelValues(dbSecret.Name, dbSecret.Namespace).Set(0)
 		}
 
 		// Stop reconciliation as the item is being deleted
@@ -199,18 +203,19 @@ func (r *DbSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	creds, err := vault.GetDbCredentials(dbSecret.Spec.Vault.Role, dbSecret.Spec.Vault.Mount)
 	if err != nil {
 		r.Log.Error(err, "Failed to obtain credentials from Vault", "name", dbSecret.Name, "namespace", dbSecret.Namespace)
-		DbSecretFailures.Inc()
-		DbSecretError.WithLabelValues(dbSecret.Name, dbSecret.Namespace).SetToCurrentTime()
+		dmetrics.DbSecretFailures.Inc()
+		dmetrics.DbSecretError.WithLabelValues(dbSecret.Name, dbSecret.Namespace).SetToCurrentTime()
 		return ctrl.Result{}, err
 	}
 
 	err = r.upsertSecret(&dbSecret, creds, currentSecret)
 	if err != nil {
 		r.Log.Error(err, "Failed to create secret", "name", dbSecret.Name, "namespace", dbSecret.Namespace)
-		DbSecretFailures.Inc()
-		DbSecretError.WithLabelValues(dbSecret.Name, dbSecret.Namespace).SetToCurrentTime()
+		dmetrics.DbSecretFailures.Inc()
+		dmetrics.DbSecretError.WithLabelValues(dbSecret.Name, dbSecret.Namespace).SetToCurrentTime()
 		return ctrl.Result{}, nil
 	}
+
 	/* Patching resources to force a rollout if required */
 	for target := range dbSecret.Spec.Rollout {
 		if dbSecret.Spec.Rollout[target].Name != "" && dbSecret.Spec.Rollout[target].Kind != "" {
@@ -375,6 +380,13 @@ func (r *DbSecretReconciler) upsertSecret(sDef *digitalisiov1beta1.DbSecret, cre
 		}
 		return err
 	}
+	/* Prometheus */
+	f, err := strconv.ParseFloat(secret.Annotations[expiresOnLabel], 10)
+	if err != nil {
+		f = float64(time.Now().UnixNano())
+	}
+	dmetrics.DbSecretExpireTime.WithLabelValues(secret.Name, secret.Namespace).Set(f)
+	dmetrics.DbSecretInfo.WithLabelValues(secret.Name, secret.Namespace).SetToCurrentTime()
 
 	if r.recordingEnabled(sDef) {
 		r.Recorder.Event(sDef, corev1.EventTypeNormal, "Updated", "Secret created or updated")

controllers/suite_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2021 Digitalis.IO.
+Copyright 2023 Digitalis.IO.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

controllers/valssecret_controller.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2022 Digitalis.IO.
+Copyright 2023 Digitalis.IO.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -44,6 +44,7 @@ import (
 	secretv1 "digitalis.io/vals-operator/apis/digitalis.io/v1"
 	valsDb "digitalis.io/vals-operator/db"
 	dbType "digitalis.io/vals-operator/db/types"
+	dmetrics "digitalis.io/vals-operator/metrics"
 	"digitalis.io/vals-operator/utils"
 	sprig "github.com/Masterminds/sprig/v3"
 )
@@ -120,6 +121,7 @@ func (r *ValsSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 		// Stop reconciliation as the item is being deleted
 		r.Log.Info(fmt.Sprintf("Secret %s deleted", secret.Name))
+		dmetrics.SecretInfo.WithLabelValues(secret.Name, secret.Namespace).Set(0)
 		return ctrl.Result{}, nil
 	}
 	//! [finalizer]
@@ -155,8 +157,11 @@ func (r *ValsSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		}
 	}
 
+	start := time.Now() // Get the current time
 	valsRendered, err := vals.Eval(secretYaml, vals.Options{})
+	elapsedPull := time.Since(start).Milliseconds() // Calculate the elapsed time
 	if err != nil {
+		dmetrics.SecretError.WithLabelValues(secret.Name, secret.Namespace).SetToCurrentTime()
 		r.Log.Error(err, "Failed to get secrets from secrets store", "name", secret.Name)
 		if r.recordingEnabled(&secret) {
 			msg := fmt.Sprintf("Failed to get secrets from secrets store %v", err)
@@ -165,14 +170,16 @@ func (r *ValsSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 		return r.errorBackoff(&secret)
 	}
+	dmetrics.SecretRetrieveTime.WithLabelValues(secret.GetName(), secret.GetNamespace()).Set(float64(elapsedPull))
 
 	data := make(map[string][]byte)
 	dataStr := make(map[string]string)
 	for k, v := range valsRendered {
 		if secret.Spec.Data[k].Encoding == "base64" && !strings.HasPrefix(secret.Spec.Data[k].Ref, k8sSecretPrefix) {
 			sDec, err := b64.StdEncoding.DecodeString(v.(string))
 			if err != nil {
-				r.Log.Error(err, "Cannot b64 decode secret. Please check encoding configuration. Requeuing.", "name", secret.Name)
+				dmetrics.SecretError.WithLabelValues(secret.Name, secret.Namespace).SetToCurrentTime()
+				r.Log.Error(err, "Cannot b64 decode secret. Please check encoding configuration. Requeuing.", "name", secret.Name, "namespace", secret.Namespace)
 				if r.recordingEnabled(&secret) {
 					r.Recorder.Event(&secret, corev1.EventTypeNormal, "Failed", "Base64 decoding failed")
 				}
@@ -192,6 +199,7 @@ func (r *ValsSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		b := bytes.NewBuffer(nil)
 		t, err := template.New(k).Funcs(sprig.FuncMap()).Parse(v)
 		if err != nil {
+			dmetrics.SecretError.WithLabelValues(secret.Name, secret.Namespace).SetToCurrentTime()
 			r.Log.Error(err, "Cannot parse template")
 			if r.recordingEnabled(&secret) {
 				msg := fmt.Sprintf("Template could not be parsed: %v", err)
@@ -200,6 +208,7 @@ func (r *ValsSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			continue
 		}
 		if err := t.Execute(b, &dataStr); err != nil {
+			dmetrics.SecretError.WithLabelValues(secret.Name, secret.Namespace).SetToCurrentTime()
 			r.Log.Error(err, "Cannot render template")
 			if r.recordingEnabled(&secret) {
 				msg := fmt.Sprintf("Template could not be rendered: %v", err)
@@ -216,6 +225,9 @@ func (r *ValsSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		r.Log.Error(err, "Failed to create secret")
 		return ctrl.Result{}, nil
 	}
+	elapsedProcess := time.Since(start).Milliseconds() // Calculate the elapsed time
+	dmetrics.SecretCreationTime.WithLabelValues(secret.GetName(), secret.GetNamespace()).Set(float64(elapsedProcess))
+	dmetrics.SecretError.WithLabelValues(secret.Name, secret.Namespace).Set(0)
 
 	r.clearErrorCount(&secret)
 	return ctrl.Result{RequeueAfter: r.ReconciliationPeriod}, nil
@@ -300,11 +312,14 @@ func (r *ValsSecretReconciler) upsertSecret(sDef *secretv1.ValsSecret, data map[
 			msg := fmt.Sprintf("Secret %s not saved %v", secret.Name, err)
 			r.Recorder.Event(sDef, corev1.EventTypeNormal, "Failed", msg)
 		}
-		SecretFailures.Inc()
-		SecretError.WithLabelValues(secret.Name, secret.Namespace).SetToCurrentTime()
+		dmetrics.SecretFailures.Inc()
+		dmetrics.SecretError.WithLabelValues(secret.Name, secret.Namespace).SetToCurrentTime()
 		return err
 	}
 
+	/* Prometheus */
+	dmetrics.SecretInfo.WithLabelValues(secret.Name, secret.Namespace).SetToCurrentTime()
+
 	if r.recordingEnabled(sDef) {
 		r.Recorder.Event(sDef, corev1.EventTypeNormal, "Updated", "Secret created or updated")
 	}

main.go

@@ -38,6 +38,7 @@ import (
 	secretv1 "digitalis.io/vals-operator/apis/digitalis.io/v1"
 	digitalisiov1beta1 "digitalis.io/vals-operator/apis/digitalis.io/v1beta1"
 	"digitalis.io/vals-operator/controllers"
+	dmetrics "digitalis.io/vals-operator/metrics"
 	"digitalis.io/vals-operator/vault"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	//+kubebuilder:scaffold:imports
@@ -56,7 +57,19 @@ func init() {
 	utilruntime.Must(secretv1.AddToScheme(scheme))
 	utilruntime.Must(digitalisiov1beta1.AddToScheme(scheme))
 
-	metrics.Registry.MustRegister(controllers.SecretFailures, controllers.DbSecretFailures, controllers.SecretError, controllers.DbSecretError)
+	metrics.Registry.MustRegister(
+		dmetrics.SecretFailures,
+		dmetrics.DbSecretFailures,
+		dmetrics.SecretError,
+		dmetrics.DbSecretError,
+		dmetrics.DbSecretExpireTime,
+		dmetrics.DbSecretInfo,
+		dmetrics.SecretInfo,
+		dmetrics.VaultError,
+		dmetrics.VaultTokenError,
+		dmetrics.SecretRetrieveTime,
+		dmetrics.SecretCreationTime,
+	)
 	//+kubebuilder:scaffold:scheme
 }