feat: add globalcontext controller (kyverno#9601)

* feat: add globalcontext controller Signed-off-by: Khaled Emara <khaled.emara@nirmata.com> * rework controller Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * rbac Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * cmd Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * fix rbac Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * engine Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * k8s resources Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * k8s resource Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * resync zero Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * api call Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * api call Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * clean Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * fix linter Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> --------- Signed-off-by: Khaled Emara <khaled.emara@nirmata.com> Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
infracloudio · Feb 2, 2024 · 226fa95 · 226fa95
1 parent 3510998
commit 226fa95
Show file tree

Hide file tree

Showing 22 changed files with 611 additions and 19 deletions.
diff --git a/charts/kyverno/templates/admission-controller/clusterrole.yaml b/charts/kyverno/templates/admission-controller/clusterrole.yaml
@@ -74,6 +74,13 @@ rules:
       - update
       - watch
       - deletecollection
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - globalcontextentries
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - reports.kyverno.io
     resources:

diff --git a/charts/kyverno/templates/background-controller/clusterrole.yaml b/charts/kyverno/templates/background-controller/clusterrole.yaml
@@ -41,6 +41,13 @@ rules:
       - update
       - watch
       - deletecollection
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - globalcontextentries
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - ''
     resources:

diff --git a/charts/kyverno/templates/cleanup-controller/clusterrole.yaml b/charts/kyverno/templates/cleanup-controller/clusterrole.yaml
@@ -51,6 +51,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - globalcontextentries
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - kyverno.io
     resources:

diff --git a/charts/kyverno/templates/reports-controller/clusterrole.yaml b/charts/kyverno/templates/reports-controller/clusterrole.yaml
@@ -34,6 +34,13 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - globalcontextentries
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - kyverno.io
     resources:

diff --git a/cmd/background-controller/main.go b/cmd/background-controller/main.go
@@ -115,6 +115,7 @@ func main() {
 		internal.WithKyvernoDynamicClient(),
 		internal.WithEventsClient(),
 		internal.WithApiServerClient(),
+		internal.WithGlobalContext(),
 		internal.WithFlagSets(flagset),
 	)
 	// parse flags

diff --git a/cmd/cleanup-controller/main.go b/cmd/cleanup-controller/main.go
@@ -19,7 +19,9 @@ import (
 	"github.com/kyverno/kyverno/pkg/controllers/cleanup"
 	genericloggingcontroller "github.com/kyverno/kyverno/pkg/controllers/generic/logging"
 	genericwebhookcontroller "github.com/kyverno/kyverno/pkg/controllers/generic/webhook"
+	"github.com/kyverno/kyverno/pkg/controllers/globalcontext"
 	ttlcontroller "github.com/kyverno/kyverno/pkg/controllers/ttl"
+	globalcontextstore "github.com/kyverno/kyverno/pkg/engine/globalcontext/store"
 	"github.com/kyverno/kyverno/pkg/event"
 	"github.com/kyverno/kyverno/pkg/informers"
 	"github.com/kyverno/kyverno/pkg/leaderelection"
@@ -101,6 +103,7 @@ func main() {
 		internal.WithDeferredLoading(),
 		internal.WithMetadataClient(),
 		internal.WithApiServerClient(),
+		internal.WithGlobalContext(),
 		internal.WithFlagSets(flagset),
 	)
 	// parse flags
@@ -156,6 +159,16 @@ func main() {
 		eventGenerator,
 		event.Workers,
 	)
+	store := globalcontextstore.New()
+	gceController := internal.NewController(
+		globalcontext.ControllerName,
+		globalcontext.NewController(
+			kyvernoInformer.Kyverno().V2alpha1().GlobalContextEntries(),
+			setup.KyvernoDynamicClient,
+			store,
+		),
+		globalcontext.Workers,
+	)
 	// start informers and wait for cache sync
 	if !internal.StartInformersAndWaitForCacheSync(ctx, setup.Logger, kubeInformer, kyvernoInformer) {
 		os.Exit(1)
@@ -349,6 +362,7 @@ func main() {
 	defer server.Stop()
 	// start non leader controllers
 	eventController.Run(ctx, setup.Logger, &wg)
+	gceController.Run(ctx, setup.Logger, &wg)
 	// start leader election
 	le.Run(ctx)
 	// wait for everything to shut down and exit

diff --git a/cmd/internal/config.go b/cmd/internal/config.go
@@ -22,6 +22,7 @@ type Configuration interface {
 	UsesMetadataClient() bool
 	UsesKyvernoDynamicClient() bool
 	UsesEventsClient() bool
+	UsesGlobalContext() bool
 	FlagSets() []*flag.FlagSet
 }
 
@@ -139,6 +140,12 @@ func WithEventsClient() ConfigurationOption {
 	}
 }
 
+func WithGlobalContext() ConfigurationOption {
+	return func(c *configuration) {
+		c.usesGlobalContext = true
+	}
+}
+
 func WithFlagSets(flagsets ...*flag.FlagSet) ConfigurationOption {
 	return func(c *configuration) {
 		c.flagSets = append(c.flagSets, flagsets...)
@@ -163,6 +170,7 @@ type configuration struct {
 	usesMetadataClient       bool
 	usesKyvernoDynamicClient bool
 	usesEventsClient         bool
+	usesGlobalContext        bool
 	flagSets                 []*flag.FlagSet
 }
 
@@ -234,6 +242,10 @@ func (c *configuration) UsesEventsClient() bool {
 	return c.usesEventsClient
 }
 
+func (c *configuration) UsesGlobalContext() bool {
+	return c.usesGlobalContext
+}
+
 func (c *configuration) FlagSets() []*flag.FlagSet {
 	return c.flagSets
 }
diff --git a/cmd/internal/flag.go b/cmd/internal/flag.go
@@ -57,6 +57,8 @@ var (
 	imageVerifyCacheEnabled     bool
 	imageVerifyCacheTTLDuration time.Duration
 	imageVerifyCacheMaxSize     int64
+	// global context
+	enableGlobalContext bool
 )
 
 func initLoggingFlags() {
@@ -135,6 +137,10 @@ func initCleanupFlags() {
 	flag.StringVar(&cleanupServerPort, "cleanupServerPort", "9443", "kyverno cleanup server port, defaults to '9443'.")
 }
 
+func initGlobalContextFlags() {
+	flag.BoolVar(&enableGlobalContext, "enableGlobalContext", true, "Enable global context feature.")
+}
+
 type options struct {
 	clientRateLimitQPS   float64
 	clientRateLimitBurst int
@@ -218,6 +224,10 @@ func initFlags(config Configuration, opts ...Option) {
 	if config.UsesLeaderElection() {
 		initLeaderElectionFlags()
 	}
+	// leader election
+	if config.UsesGlobalContext() {
+		initGlobalContextFlags()
+	}
 	initCleanupFlags()
 	for _, flagset := range config.FlagSets() {
 		flagset.VisitAll(func(f *flag.Flag) {
@@ -255,6 +265,10 @@ func CleanupServerPort() string {
 	return cleanupServerPort
 }
 
+func GlobalContextEnabled() bool {
+	return enableGlobalContext
+}
+
 func printFlagSettings(logger logr.Logger) {
 	logger = logger.WithName("flag")
 	flag.VisitAll(func(f *flag.Flag) {

diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go
@@ -258,6 +258,7 @@ func main() {
 		internal.WithKyvernoDynamicClient(),
 		internal.WithEventsClient(),
 		internal.WithApiServerClient(),
+		internal.WithGlobalContext(),
 		internal.WithFlagSets(flagset),
 	)
 	// parse flags

diff --git a/cmd/reports-controller/main.go b/cmd/reports-controller/main.go
@@ -242,6 +242,7 @@ func main() {
 		internal.WithKyvernoDynamicClient(),
 		internal.WithEventsClient(),
 		internal.WithApiServerClient(),
+		internal.WithGlobalContext(),
 		internal.WithFlagSets(flagset),
 	)
 	// parse flags

diff --git a/config/install-latest-testing.yaml b/config/install-latest-testing.yaml
@@ -50588,6 +50588,13 @@ rules:
       - update
       - watch
       - deletecollection
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - globalcontextentries
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - reports.kyverno.io
     resources:
@@ -50711,6 +50718,13 @@ rules:
       - update
       - watch
       - deletecollection
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - globalcontextentries
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - ''
     resources:
@@ -50833,6 +50847,13 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - globalcontextentries
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - kyverno.io
     resources:
@@ -51137,6 +51158,13 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - globalcontextentries
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - kyverno.io
     resources:

diff --git a/pkg/controllers/globalcontext/controller.go b/pkg/controllers/globalcontext/controller.go
@@ -0,0 +1,107 @@
+package globalcontext
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/go-logr/logr"
+	kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1"
+	kyvernov2alpha1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v2alpha1"
+	kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1"
+	"github.com/kyverno/kyverno/pkg/clients/dclient"
+	"github.com/kyverno/kyverno/pkg/controllers"
+	"github.com/kyverno/kyverno/pkg/engine/adapters"
+	"github.com/kyverno/kyverno/pkg/engine/globalcontext/externalapi"
+	"github.com/kyverno/kyverno/pkg/engine/globalcontext/k8sresource"
+	"github.com/kyverno/kyverno/pkg/engine/globalcontext/store"
+	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/util/workqueue"
+)
+
+const (
+	// Workers is the number of workers for this controller
+	Workers        = 1
+	ControllerName = "global-context"
+	maxRetries     = 10
+)
+
+type controller struct {
+	// listers
+	gceLister kyvernov2alpha1listers.GlobalContextEntryLister
+
+	// queue
+	queue workqueue.RateLimitingInterface
+
+	// state
+	dclient dclient.Interface
+	store   store.Store
+}
+
+func NewController(
+	gceInformer kyvernov2alpha1informers.GlobalContextEntryInformer,
+	dclient dclient.Interface,
+	storage store.Store,
+) controllers.Controller {
+	queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName)
+	_, _, err := controllerutils.AddDefaultEventHandlers(logger, gceInformer.Informer(), queue)
+	if err != nil {
+		logger.Error(err, "failed to register event handlers")
+	}
+	return &controller{
+		gceLister: gceInformer.Lister(),
+		queue:     queue,
+		dclient:   dclient,
+		store:     storage,
+	}
+}
+
+func (c *controller) Run(ctx context.Context, workers int) {
+	controllerutils.Run(ctx, logger, ControllerName, time.Second, c.queue, workers, maxRetries, c.reconcile)
+}
+
+func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, name string) error {
+	gce, err := c.getEntry(name)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			// entry was deleted, remove it from the store
+			c.store.Delete(name)
+			return nil
+		}
+		return err
+	}
+	// either it's a new entry or an existing entry changed
+	// create a new element and set it in the store
+	entry, err := c.makeStoreEntry(ctx, gce)
+	if err != nil {
+		return err
+	}
+	c.store.Set(name, entry)
+	return nil
+}
+
+func (c *controller) getEntry(name string) (*kyvernov2alpha1.GlobalContextEntry, error) {
+	return c.gceLister.Get(name)
+}
+
+func (c *controller) makeStoreEntry(ctx context.Context, gce *kyvernov2alpha1.GlobalContextEntry) (store.Entry, error) {
+	// TODO: should be done at validation time
+	if gce.Spec.KubernetesResource == nil && gce.Spec.APICall == nil {
+		return nil, errors.New("global context entry neither has K8sResource nor APICall")
+	}
+	// TODO: should be done at validation time
+	if gce.Spec.KubernetesResource != nil && gce.Spec.APICall != nil {
+		return nil, errors.New("global context entry has both K8sResource and APICall")
+	}
+	if gce.Spec.KubernetesResource != nil {
+		gvr := schema.GroupVersionResource{
+			Group:    gce.Spec.KubernetesResource.Group,
+			Version:  gce.Spec.KubernetesResource.Version,
+			Resource: gce.Spec.KubernetesResource.Resource,
+		}
+		return k8sresource.New(ctx, c.dclient.GetDynamicInterface(), gvr, gce.Spec.KubernetesResource.Namespace)
+	}
+	return externalapi.New(ctx, logger, adapters.Client(c.dclient), gce.Spec.APICall.APICall, time.Duration(gce.Spec.APICall.RefreshIntervalSeconds))
+}
diff --git a/pkg/controllers/globalcontext/log.go b/pkg/controllers/globalcontext/log.go
@@ -0,0 +1,5 @@
+package globalcontext
+
+import "github.com/kyverno/kyverno/pkg/logging"
+
+var logger = logging.ControllerLogger(ControllerName)