Merge pull request #1576 from flanksource/k8s-fixs

fix: kubernetes checks

.gitignore

@@ -25,3 +25,7 @@ tmp/
 ginkgo.report
 __debug*
 test-results.xml
+coverprofile.out
+junit-report.xml
+canary-checker.properties
+*.code-workspace

Makefile

@@ -26,9 +26,10 @@ endif
 
 all: manager
 
-# Run tests
-test: generate fmt vet manifests
-	go test ./... -coverprofile cover.out
+
+.PHONY: test
+test: manifests generate fmt ginkgo
+	ginkgo -vv -r  --cover  --keep-going --junit-report junit-report.xml --
 
 # Build manager binary
 manager: generate fmt vet
@@ -121,7 +122,7 @@ docker-push:
 
 .PHONY: compress
 compress: .bin/upx
-	upx -5 ./.bin/$(NAME)_linux_amd64 ./.bin/$(NAME)_linux_arm64 ./.bin/$(NAME).exe
+	upx -5 ./.bin/$(NAME)_linux_amd64 ./.bin/$(NAME)_linux_arm64
 
 .PHONY: compress-build
 compress-build: .bin/upx
@@ -201,6 +202,11 @@ else
 	UPX=$(shell which upx)
 endif
 
+
+.PHONY: ginkgo
+ginkgo:
+	go install github.com/onsi/ginkgo/v2/ginkgo
+
 .bin/controller-gen: .bin
 		GOBIN=$(PWD)/.bin go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.1
 		CONTROLLER_GEN=$(GOBIN)/controller-gen
@@ -227,25 +233,10 @@ endif
 	curl -sSLo .bin/karina https://github.com/flanksource/karina/releases/download/v0.50.0/karina_$(OS)-$(ARCH) && \
 	chmod +x .bin/karina
 
-.PHONY: telepresence
-telepresence:
-ifeq (, $(shell which telepresence))
-ifeq ($(OS), darwin)
-	brew install --cask macfuse
-	brew install datawire/blackbird/telepresence-legacy
-else
-	sudo apt-get install -y conntrack
-	wget https://s3.amazonaws.com/datawire-static-files/telepresence/telepresence-0.109.tar.gz
-	tar xzf telepresence-0.109.tar.gz
-	sudo mv telepresence-0.109/bin/telepresence /usr/local/bin/
-	sudo mv telepresence-0.109/libexec/sshuttle-telepresence /usr/local/bin/
-endif
-endif
-
 .bin:
 	mkdir -p .bin
 
-bin: .bin .bin/wait4x .bin/karina .bin/go-junit-report  telepresence
+bin: .bin .bin/wait4x .bin/karina
 
 # Generate all the resources and formats your code, i.e: CRDs, controller-gen, static
 .PHONY: resources

api/context/context.go

@@ -10,6 +10,7 @@ import (
 	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
+	"github.com/flanksource/commons/console"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty/connection"
 	dutyCtx "github.com/flanksource/duty/context"
@@ -18,6 +19,7 @@ import (
 	"github.com/flanksource/gomplate/v3"
 	"github.com/flanksource/kommons"
 	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/samber/lo"
 	"gorm.io/gorm"
 	"k8s.io/client-go/kubernetes"
 )
@@ -262,8 +264,8 @@ func New(ctx dutyCtx.Context, canary v1.Canary) *Context {
 		canary.Namespace = "default"
 	}
 
-	return &Context{
-		duty:        &ctx,
+	c := &Context{
+		duty:        lo.ToPtr(ctx.WithObject(canary.ObjectMeta)),
 		db:          ctx.DB(),
 		pool:        ctx.Pool(),
 		Context:     gocontext.Background(),
@@ -274,25 +276,34 @@ func New(ctx dutyCtx.Context, canary v1.Canary) *Context {
 		Environment: make(map[string]interface{}),
 		Logger:      logger.StandardLogger(),
 	}
+
+	c.Logger = c.Logger.Named(console.DarkWhitef("canary.%s.%s", canary.Namespace, canary.Name))
+
+	if c.Logger.IsLevelEnabled(4) || c.duty.IsTrace() {
+		c.Logger.SetMinLogLevel(2)
+	} else if c.Logger.IsLevelEnabled(3) || c.duty.IsDebug() {
+		c.Logger.SetMinLogLevel(1)
+	}
+	return c
 }
 
 func (ctx *Context) IsDebug() bool {
-	return ctx.Canary.IsDebug() || ctx.IsTrace()
+	return ctx.Logger.IsLevelEnabled(3) || ctx.Canary.IsDebug() || ctx.IsTrace()
 }
 
 func (ctx *Context) IsTrace() bool {
-	return ctx.Canary.IsTrace()
+	return ctx.Logger.IsLevelEnabled(4) || ctx.Canary.IsTrace()
 }
 
 func (ctx *Context) Debugf(format string, args ...interface{}) {
 	if ctx.IsDebug() {
-		ctx.Logger.Infof(format, args...)
+		ctx.Logger.Debugf(format, args...)
 	}
 }
 
 func (ctx *Context) Tracef(format string, args ...interface{}) {
 	if ctx.IsTrace() {
-		ctx.Logger.Infof(format, args...)
+		ctx.Logger.Tracef(format, args...)
 	}
 }
 

checks/common.go

@@ -141,7 +141,7 @@ func transform(ctx *context.Context, in *pkg.CheckResult) ([]*pkg.CheckResult, b
 			results = append(results, &r)
 		}
 		if ctx.IsTrace() {
-			ctx.Tracef("transformed %s into %v", in, results)
+			ctx.Tracef("transformed %s into %d results", in, len(results))
 		}
 		return results, hasTransformer, nil
 	} else if len(transformed) == 1 && t.Name == "" {

checks/kubernetes.go

@@ -68,6 +68,9 @@ func (c *KubernetesChecker) Check(ctx *context.Context, extConfig external.Check
 			if err == nil {
 				resource.Object["healthStatus"] = resourceHealth
 			}
+			if check.Ready != nil && *check.Ready && resourceHealth.Status != health.HealthStatusHealthy {
+				results.Failf("%s/%s/%s is %s: %s", resource.GetKind(), resource.GetNamespace(), resource.GetName(), resourceHealth.Status, resourceHealth.Message)
+			}
 		}
 		allResources = append(allResources, resources...)
 	}

cmd/operator.go

@@ -12,6 +12,7 @@ import (
 	"github.com/flanksource/canary-checker/pkg/runner"
 	"github.com/flanksource/canary-checker/pkg/utils"
 	gocache "github.com/patrickmn/go-cache"
+	"go.uber.org/zap/zapcore"
 
 	canaryv1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
@@ -31,6 +32,7 @@ import (
 )
 
 var webhookPort int
+var k8sLogLevel int
 var enableLeaderElection bool
 var operatorExecutor bool
 var disablePostgrest bool
@@ -45,6 +47,7 @@ func init() {
 	Operator.Flags().StringVarP(&runner.WatchNamespace, "namespace", "n", "", "Watch only specified namespaces, otherwise watch all")
 	Operator.Flags().BoolVar(&operatorExecutor, "executor", true, "If false, only serve the UI and sync the configs")
 	Operator.Flags().IntVar(&webhookPort, "webhookPort", 8082, "Port for webhooks ")
+	Operator.Flags().IntVar(&k8sLogLevel, "k8s-log-level", -1, "Kubernetes controller log level")
 	Operator.Flags().BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enabling this will ensure there is only one active controller manager")
 	Operator.Flags().BoolVar(&disablePostgrest, "disable-postgrest", false, "Disable the postgrest server")
 	// +kubebuilder:scaffold:scheme
@@ -58,12 +61,9 @@ func run(cmd *cobra.Command, args []string) {
 	}
 
 	loggr := ctrlzap.NewRaw(
-		ctrlzap.UseDevMode(true),
-		ctrlzap.WriteTo(os.Stderr),
-		ctrlzap.Level(zapLogger.Level),
-		ctrlzap.StacktraceLevel(zapLogger.StackTraceLevel),
-		ctrlzap.Encoder(zapLogger.GetEncoder()),
-	)
+		ctrlzap.Encoder(logger.NewZapEncoder()),
+		ctrlzap.Level(zapcore.Level(k8sLogLevel*-1)),
+	).Named("operator")
 
 	scheme := runtime.NewScheme()
 

cmd/root.go

@@ -47,10 +47,10 @@ func InitContext() (context.Context, error) {
 var Root = &cobra.Command{
 	Use: "canary-checker",
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
-		canary.LogFail = logFail
-		canary.LogPass = logPass
+		logger.UseZap()
 
-		logger.UseZap(cmd.Flags())
+		canary.LogFail = logFail || logger.IsLevelEnabled(3)
+		canary.LogPass = logPass || logger.IsLevelEnabled(4)
 
 		db.ConnectionString = readFromEnv(db.ConnectionString)
 		if db.ConnectionString == "DB_URL" {
@@ -59,7 +59,7 @@ var Root = &cobra.Command{
 
 		if canary.UpstreamConf.Valid() {
 			logger.Infof("Pushing checks %s", canary.UpstreamConf)
-		} else {
+		} else if canary.UpstreamConf.IsPartiallyFilled() {
 			logger.Debugf("Upstream not fully configured: %s", canary.UpstreamConf)
 		}
 

cmd/run.go

@@ -2,18 +2,15 @@ package cmd
 
 import (
 	"encoding/json"
-	"fmt"
 	"log"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/flanksource/commons/timer"
 
 	"github.com/flanksource/canary-checker/cmd/output"
-	"github.com/flanksource/canary-checker/pkg/db"
 	"github.com/spf13/cobra"
 
 	apicontext "github.com/flanksource/canary-checker/api/context"
@@ -28,11 +25,6 @@ var junit, csv, jsonExport bool
 var Run = &cobra.Command{
 	Use:   "run <canary.yaml>",
 	Short: "Execute checks and return",
-	PersistentPreRun: func(cmd *cobra.Command, args []string) {
-		logger.ParseFlags(cmd.Flags())
-		db.ConnectionString = readFromEnv(db.ConnectionString)
-
-	},
 	Run: func(cmd *cobra.Command, configFiles []string) {
 		timer := timer.NewTimer()
 		if len(configFiles) == 0 {
@@ -90,7 +82,12 @@ var Run = &cobra.Command{
 				} else {
 					passed++
 				}
-				fmt.Printf("%s \t%s\t\n", time.Now().Format(time.RFC3339), result.String())
+				logPass := result.Canary.IsDebug() || result.Canary.IsTrace() || logPass
+				logFail := result.Canary.IsDebug() || result.Canary.IsTrace() || logFail
+
+				if logPass && result.Pass || logFail && !result.Pass {
+					logger.Infof(result.String())
+				}
 				results = append(results, result)
 			}
 		}

cmd/serve.go

@@ -39,7 +39,6 @@ var Serve = &cobra.Command{
 	Use:   "serve config.yaml",
 	Short: "Start a server to execute checks",
 	Run: func(cmd *cobra.Command, configFiles []string) {
-		logger.ParseFlags(cmd.Flags())
 		canaryJobs.StartScanCanaryConfigs(setup(), dataFile, configFiles)
 		if executor {
 			jobs.Start()

cmd/topology.go

@@ -26,7 +26,6 @@ var topologyRunNamespace string
 var Topology = &cobra.Command{
 	Use: "topology",
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
-		logger.ParseFlags(cmd.Flags())
 		db.ConnectionString = readFromEnv(db.ConnectionString)
 		var err error
 		apicontext.DefaultContext, err = InitContext()

fixtures/git/_image

@@ -0,0 +1 @@
+flanksource/canary-checker-full

fixtures/git/_setup.sh

@@ -38,4 +38,4 @@ curl -vvv  -u gitea_admin:admin   -H "Content-Type: application/json"  http://lo
 kill $PID
 
 
-kubectl create secret generic gitea --from-literal=username=gitea_admin --from-literal=password=admin --from-literal=url=http:///gitea-http.gitea.svc:3000/gitea_admin/test_repo.git
+kubectl create secret generic gitea --from-literal=username=gitea_admin --from-literal=password=admin --from-literal=url=http:///gitea-http.gitea.svc:3000/gitea_admin/test_repo.git --namespace canaries

fixtures/k8s/kubernetes-minimal_pass.yaml

@@ -0,0 +1,23 @@
+apiVersion: canaries.flanksource.com/v1
+kind: Canary
+metadata:
+  name: kube-system-checks
+spec:
+  interval: 30
+  kubernetes:
+    - namespace: kube-system
+      name: kube-system
+      kind: Pod
+      # ready: true
+      # resource:
+      #   labelSelector: k8s-app=kube-dns
+      namespaceSelector:
+        name: default
+      display:
+        expr: |
+          dyn(results).
+          map(i, i.Object).
+          filter(i, !k8s.isHealthy(i)).
+          map(i, "%s/%s -> %s".format([i.metadata.namespace, i.metadata.name, k8s.getHealth(i).message])).join('\n')
+      test:
+        expr: dyn(results).all(x, k8s.isHealthy(x))

fixtures/k8s/kubernetes_bundle.yaml

@@ -12,6 +12,7 @@ spec:
         expr: |
           dyn(results).map(r, {
             'name': r.Object.metadata.name,
+            'namespace': r.Object.metadata.?namespace.orValue(null),
             'labels': r.Object.metadata.labels,
             'pass': k8s.isHealthy(r.Object),
             'message': k8s.getHealth(r.Object).message,
@@ -20,10 +21,13 @@ spec:
     - kind: Pod
       ready: true
       name: pod-bundle
+      resource:
+        labelSelector: app != k8s-not-ready, app != k8s-ready, Expected-Fail != true, canary-checker.flanksource.com/generated != true, !canary-checker.flanksource.com/check
       transform:
         expr: |
           dyn(results).map(r, {
             'name': r.Object.metadata.name,
+            'namespace': r.Object.metadata.namespace,
             'labels': r.Object.metadata.labels,
             'pass': k8s.isHealthy(r.Object),
             'message': k8s.getHealth(r.Object).message,

fixtures/k8s/namespace_pass.yaml → fixtures/k8s/slow/namespace_pass.yaml

@@ -23,9 +23,9 @@ spec:
       ingressName: test-namespace-pod
       ingressHost: "test-namespace-pod.127.0.0.1.nip.io"
       readyTimeout: 5000
-      httpTimeout: 15000
+      httpTimeout: 40000
       deleteTimeout: 12000
-      ingressTimeout: 20000
+      ingressTimeout: 40000
       deadline: 60000
       httpRetryInterval: 1500
       expectedContent: bar

fixtures/restic/_image

@@ -0,0 +1 @@
+flanksource/canary-checker-full

go.mod

@@ -21,8 +21,8 @@ require (
 	github.com/elastic/go-elasticsearch/v8 v8.11.1
 	github.com/fergusstrange/embedded-postgres v1.25.0
 	github.com/flanksource/artifacts v1.0.4
-	github.com/flanksource/commons v1.20.0
-	github.com/flanksource/duty v1.0.287
+	github.com/flanksource/commons v1.20.3
+	github.com/flanksource/duty v1.0.290
 	github.com/flanksource/gomplate/v3 v3.21.0
 	github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7
 	github.com/flanksource/kommons v0.31.4
@@ -69,6 +69,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0
 	go.opentelemetry.io/otel/sdk v1.21.0
 	go.opentelemetry.io/otel/trace v1.21.0
+	go.uber.org/zap v1.26.0
 	golang.org/x/net v0.19.0
 	golang.org/x/sync v0.6.0
 	google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917
@@ -244,7 +245,6 @@ require (
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
 	golang.org/x/oauth2 v0.15.0 // indirect
@@ -280,9 +280,3 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.16.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
-
-// replace github.com/flanksource/duty => ../duty
-
-// replace github.com/flanksource/gomplate/v3 => ../gomplate
-
-// replace github.com/flanksource/artifacts => ../artifacts

go.sum

@@ -825,10 +825,10 @@ github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7Ttk
 github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw=
 github.com/flanksource/artifacts v1.0.4 h1:KjQTwsvQ73uHqTK7o4Jwt/RW8fyxJOTJ6JLgzrGTV7w=
 github.com/flanksource/artifacts v1.0.4/go.mod h1:wkbdseaTkDo4Q6k6T86vXd4Uy47M6NPCmexgHvCTDl0=
-github.com/flanksource/commons v1.20.0 h1:1z7FHGsCY7F7zMx4m0OlzYCUvCc1caREPp7vf1xyeCQ=
-github.com/flanksource/commons v1.20.0/go.mod h1:bs2nMwaTCpTQGZNtJv8KNqEVyfGjFZTGUDaAbPHB1cw=
-github.com/flanksource/duty v1.0.287 h1:vuXyoxoTkEWh2m08d3JcaB/ZkFuTAEmLwBHQuW5e2AE=
-github.com/flanksource/duty v1.0.287/go.mod h1:zDVxlWxpSZjbpuBv0i9ke4vlG2elgyKjYY+656XhfPs=
+github.com/flanksource/commons v1.20.3 h1:8HQvskTl6yghQIEMuLs4RWUtt9NS60MPD/iyG32VrYI=
+github.com/flanksource/commons v1.20.3/go.mod h1:bs2nMwaTCpTQGZNtJv8KNqEVyfGjFZTGUDaAbPHB1cw=
+github.com/flanksource/duty v1.0.290 h1:s2PkWbMXEXcpL/Ei3gFo7cUq3P7nHK2/LINtEz6PgJg=
+github.com/flanksource/duty v1.0.290/go.mod h1:CpUK3Hh9tMSRcotQhdQCYOE/VRqT2uy3IIGpH2DWzoY=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.21.0 h1:HbgJ7GCv2fGjjKhGBBtMmo+uA9T6pI9FVm+M/eEurOA=
 github.com/flanksource/gomplate/v3 v3.21.0/go.mod h1:m2WVc04GMVBOcZhtDaz/LTtrVWKejeJhFM1Jy/h9VZQ=