From e34eed0643513a863d5b85eb6e04fa09a3218a88 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 2 Oct 2023 18:40:27 +0000 Subject: [PATCH 01/42] add syncset controller Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/add_syncset.go | 9 + pkg/controller/syncset/syncset_controller.go | 192 +++++++++++ .../syncset/syncset_controller_test.go | 299 ++++++++++++++++++ pkg/controller/syncset/syncset_suite_test.go | 29 ++ test/testutils/controller.go | 90 +++++- 5 files changed, 613 insertions(+), 6 deletions(-) create mode 100644 pkg/controller/add_syncset.go create mode 100644 pkg/controller/syncset/syncset_controller.go create mode 100644 pkg/controller/syncset/syncset_controller_test.go create mode 100644 pkg/controller/syncset/syncset_suite_test.go diff --git a/pkg/controller/add_syncset.go b/pkg/controller/add_syncset.go new file mode 100644 index 00000000000..80c6426ac85 --- /dev/null +++ b/pkg/controller/add_syncset.go @@ -0,0 +1,9 @@ +package controller + +import ( + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/syncset" +) + +func init() { + Injectors = append(Injectors, &syncset.Adder{}) +} diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go new file mode 100644 index 00000000000..9eca69f7684 --- /dev/null +++ b/pkg/controller/syncset/syncset_controller.go @@ -0,0 +1,192 @@ +package syncset + +import ( + "context" + "fmt" + + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" + "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" + "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + ctrlName = "syncset-controller" + finalizerName = "finalizers.gatekeeper.sh/syncset" +) + +var ( + log = logf.Log.WithName("controller").WithValues("kind", "SyncSet", logging.Process, "syncset_controller") + + syncsetGVK = schema.GroupVersionKind{ + Group: syncsetv1alpha1.GroupVersion.Group, + Version: syncsetv1alpha1.GroupVersion.Version, + Kind: "SyncSet", + } +) + +type Adder struct { + CacheManager *cm.CacheManager + ControllerSwitch *watch.ControllerSwitch + Tracker *readiness.Tracker +} + +// Add creates a new SyncSetController and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func (a *Adder) Add(mgr manager.Manager) error { + r, err := newReconciler(mgr, a.CacheManager, a.ControllerSwitch, a.Tracker) + if err != nil { + return err + } + + return add(mgr, r) +} + +func (a *Adder) InjectCacheManager(o *cm.CacheManager) { + a.CacheManager = o +} + +func (a *Adder) InjectControllerSwitch(cs *watch.ControllerSwitch) { + a.ControllerSwitch = cs +} + +func (a *Adder) InjectTracker(t *readiness.Tracker) { + a.Tracker = t +} + +func newReconciler(mgr manager.Manager, cm *cm.CacheManager, cs *watch.ControllerSwitch, tracker *readiness.Tracker) (*ReconcileSyncSet, error) { + if cm == nil { + return nil, fmt.Errorf("CacheManager must be non-nil") + } + if tracker == nil { + return nil, fmt.Errorf("ReadyTracker must be non-nil") + } + + return &ReconcileSyncSet{ + reader: mgr.GetCache(), + writer: mgr.GetClient(), + scheme: mgr.GetScheme(), + cs: cs, + cacheManager: cm, + tracker: tracker, + }, nil +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + c, err := controller.New(ctrlName, mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + err = c.Watch(source.Kind(mgr.GetCache(), &syncsetv1alpha1.SyncSet{}), &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + return nil +} + +var _ reconcile.Reconciler = &ReconcileSyncSet{} + +// ReconcileSyncSet reconciles a SyncSet object. +type ReconcileSyncSet struct { + reader client.Reader + writer client.Writer + + scheme *runtime.Scheme + cacheManager *cm.CacheManager + cs *watch.ControllerSwitch + tracker *readiness.Tracker +} + +func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + // Short-circuit if shutting down. + if r.cs != nil { + running := r.cs.Enter() + defer r.cs.Exit() + if !running { + return reconcile.Result{}, nil + } + } + + syncsetTr := r.tracker.For(syncsetGVK) + exists := true + instance := &syncsetv1alpha1.SyncSet{} + err := r.reader.Get(ctx, request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + exists = false + } else { + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + } + + if exists { + // Actively remove a finalizer. This should automatically remove + // the finalizer over time even if state teardown didn't work correctly + // after a deprecation period, all finalizer code can be removed. + if hasFinalizer(instance) { + removeFinalizer(instance) + if err := r.writer.Update(ctx, instance); err != nil { + return reconcile.Result{}, err + } + } + syncsetTr.Observe(instance) + } + + gvks := make([]schema.GroupVersionKind, 0) + if exists && instance.GetDeletionTimestamp().IsZero() { + log.Info("handling SyncSet update", "instance", instance) + + for _, entry := range instance.Spec.GVKs { + gvks = append(gvks, schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind}) + } + } + + sk := aggregator.Key{Source: "syncset", ID: request.NamespacedName.String()} + if err := r.cacheManager.UpsertSource(ctx, sk, gvks); err != nil { + return reconcile.Result{Requeue: true}, fmt.Errorf("synceset-controller: error changing watches: %w", err) + } + + return reconcile.Result{}, nil +} + +func containsString(s string, items []string) bool { + for _, item := range items { + if item == s { + return true + } + } + return false +} + +func removeString(s string, items []string) []string { + var rval []string + for _, item := range items { + if item != s { + rval = append(rval, item) + } + } + return rval +} + +func hasFinalizer(instance *syncsetv1alpha1.SyncSet) bool { + return containsString(finalizerName, instance.GetFinalizers()) +} + +func removeFinalizer(instance *syncsetv1alpha1.SyncSet) { + instance.SetFinalizers(removeString(finalizerName, instance.GetFinalizers())) +} diff --git a/pkg/controller/syncset/syncset_controller_test.go b/pkg/controller/syncset/syncset_controller_test.go new file mode 100644 index 00000000000..99bf2c5edc2 --- /dev/null +++ b/pkg/controller/syncset/syncset_controller_test.go @@ -0,0 +1,299 @@ +package syncset + +import ( + "fmt" + "testing" + "time" + + constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" + "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" + "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + "github.com/open-policy-agent/gatekeeper/v3/pkg/target" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" + testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" + "github.com/open-policy-agent/gatekeeper/v3/test/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var ( + configMapGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + nsGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} + podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} +) + +const ( + timeout = time.Second * 20 + tick = time.Second * 2 +) + +// Test_ReconcileSyncSet_wConfigController verifies that SyncSet and Config resources +// can get reconciled and their respective specs are added to the data client. +func Test_ReconcileSyncSet_wConfigController(t *testing.T) { + require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) + + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + instanceConfig := testutils.ConfigFor([]schema.GroupVersionKind{}) + instanceSyncSet1 := &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "syncset1", + }, + } + instanceSyncSet2 := &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "syncset2", + }, + } + configMap := testutils.UnstructuredFor(configMapGVK, "", "cm1-name") + pod := testutils.UnstructuredFor(podGVK, "", "pod1-name") + + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + + cfClient := &fakes.FakeCfClient{} + cs := watch.NewSwitch() + tracker, err := readiness.SetupTracker(mgr, false, false, false) + if err != nil { + t.Fatal(err) + } + processExcluder := process.Get() + events := make(chan event.GenericEvent, 1024) + syncMetricsCache := syncutil.NewMetricsCache() + w, err := wm.NewRegistrar( + cm.RegistrarName, + events) + require.NoError(t, err) + + cm, err := cm.NewCacheManager(&cm.Config{ + CfClient: cfClient, + SyncMetricsCache: syncMetricsCache, + Tracker: tracker, + ProcessExcluder: processExcluder, + Registrar: w, + Reader: c, + }) + require.NoError(t, err) + go func() { + assert.NoError(t, cm.Start(ctx)) + }() + + rec, err := newReconciler(mgr, cm, cs, tracker) + require.NoError(t, err, "creating sync set reconciler") + require.NoError(t, add(mgr, rec), "adding syncset reconciler to mgr") + + // for sync controller + syncAdder := syncc.Adder{CacheManager: cm, Events: events} + require.NoError(t, syncAdder.Add(mgr), "adding sync reconciler to mgr") + + // now for config controller + configAdder := config.Adder{ + CacheManager: cm, + ControllerSwitch: cs, + Tracker: tracker, + } + require.NoError(t, configAdder.Add(mgr), "adding config reconciler to mgr") + + testutils.StartManager(ctx, t, mgr) + + require.NoError(t, c.Create(ctx, configMap), fmt.Sprintf("creating ConfigMap %s", "cm1-mame")) + require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", "pod1-name")) + + tts := []struct { + name string + setup func(t *testing.T) + cleanup func(t *testing.T) + expectedGVKs []schema.GroupVersionKind + }{ + { + name: "config and 1 sync", + setup: func(t *testing.T) { + t.Helper() + + instanceConfig := testutils.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}) + instanceSyncSet := &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "syncset1", + }, + Spec: syncsetv1alpha1.SyncSetSpec{ + GVKs: []syncsetv1alpha1.GVKEntry{ + syncsetv1alpha1.GVKEntry(podGVK), + }, + }, + } + + require.NoError(t, c.Create(ctx, instanceConfig)) + require.NoError(t, c.Create(ctx, instanceSyncSet)) + }, + cleanup: func(t *testing.T) { + t.Helper() + + // reset the sync instances + require.NoError(t, c.Delete(ctx, instanceConfig)) + require.NoError(t, c.Delete(ctx, instanceSyncSet1)) + }, + expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK, nsGVK}, + }, + { + name: "config and multiple sync", + setup: func(t *testing.T) { + t.Helper() + + instanceConfig := testutils.ConfigFor([]schema.GroupVersionKind{configMapGVK}) + instanceSyncSet1 = &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "syncset1", + }, + Spec: syncsetv1alpha1.SyncSetSpec{ + GVKs: []syncsetv1alpha1.GVKEntry{ + syncsetv1alpha1.GVKEntry(podGVK), + }, + }, + } + instanceSyncSet2 = &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "syncset2", + }, + Spec: syncsetv1alpha1.SyncSetSpec{ + GVKs: []syncsetv1alpha1.GVKEntry{ + syncsetv1alpha1.GVKEntry(configMapGVK), + }, + }, + } + + require.NoError(t, c.Create(ctx, instanceConfig)) + require.NoError(t, c.Create(ctx, instanceSyncSet1)) + require.NoError(t, c.Create(ctx, instanceSyncSet2)) + }, + cleanup: func(t *testing.T) { + t.Helper() + + // reset the sync instances + require.NoError(t, c.Delete(ctx, instanceConfig)) + require.NoError(t, c.Delete(ctx, instanceSyncSet1)) + require.NoError(t, c.Delete(ctx, instanceSyncSet2)) + }, + expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, + }, + } + + for _, tt := range tts { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup(t) + } + + assert.Eventually(t, expectedCheck(cfClient, tt.expectedGVKs), timeout, tick) + + if tt.cleanup != nil { + tt.cleanup(t) + + require.Eventually(t, func() bool { + return cfClient.Len() == 0 + }, timeout, tick, "could not cleanup") + } + }) + } + + cs.Stop() +} + +func expectedCheck(cfClient *fakes.FakeCfClient, expected []schema.GroupVersionKind) func() bool { + return func() bool { + for _, gvk := range expected { + if !cfClient.HasGVK(gvk) { + return false + } + } + return true + } +} + +func Test_ReconcileSyncSet_Reconcile(t *testing.T) { + require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) + + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + instanceSyncSet := &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "syncset", + }, + Spec: syncsetv1alpha1.SyncSetSpec{ + GVKs: []syncsetv1alpha1.GVKEntry{syncsetv1alpha1.GVKEntry(podGVK)}, + }, + } + + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + + driver, err := rego.New() + require.NoError(t, err, "unable to set up driver") + + dataClient, err := constraintclient.NewClient(constraintclient.Targets(&target.K8sValidationTarget{}), constraintclient.Driver(driver)) + require.NoError(t, err, "unable to set up data client") + + cs := watch.NewSwitch() + tracker, err := readiness.SetupTracker(mgr, false, false, false) + require.NoError(t, err) + + processExcluder := process.Get() + events := make(chan event.GenericEvent, 1024) + syncMetricsCache := syncutil.NewMetricsCache() + w, err := wm.NewRegistrar( + cm.RegistrarName, + events) + require.NoError(t, err) + + cm, err := cm.NewCacheManager(&cm.Config{CfClient: dataClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, Registrar: w, Reader: c}) + require.NoError(t, err) + go func() { + assert.NoError(t, cm.Start(ctx)) + }() + + rec, err := newReconciler(mgr, cm, cs, tracker) + require.NoError(t, err) + + recFn, requests := testutils.SetupTestReconcile(rec) + require.NoError(t, add(mgr, recFn)) + + testutils.StartManager(ctx, t, mgr) + + require.NoError(t, c.Create(ctx, instanceSyncSet)) + defer func() { + ctx := context.Background() + require.NoError(t, c.Delete(ctx, instanceSyncSet)) + }() + + require.Eventually(t, func() bool { + _, ok := requests.Load(reconcile.Request{NamespacedName: types.NamespacedName{Name: "syncset"}}) + + return ok + }, timeout, tick, "waiting on syncset request to be received") + + require.Eventually(t, func() bool { + return len(wm.GetManagedGVK()) == 1 + }, timeout, tick, "check watched gvks are populated") + + gvks := wm.GetManagedGVK() + wantGVKs := []schema.GroupVersionKind{ + {Group: "", Version: "v1", Kind: "Pod"}, + } + require.ElementsMatch(t, wantGVKs, gvks) + + cs.Stop() +} diff --git a/pkg/controller/syncset/syncset_suite_test.go b/pkg/controller/syncset/syncset_suite_test.go new file mode 100644 index 00000000000..3abeb2a2a9f --- /dev/null +++ b/pkg/controller/syncset/syncset_suite_test.go @@ -0,0 +1,29 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package syncset + +import ( + "testing" + + "github.com/open-policy-agent/gatekeeper/v3/test/testutils" + "k8s.io/client-go/rest" +) + +var cfg *rest.Config + +func TestMain(m *testing.M) { + testutils.StartControlPlane(m, &cfg, 3) +} diff --git a/test/testutils/controller.go b/test/testutils/controller.go index ddcd9f8c9a7..727a974f7bb 100644 --- a/test/testutils/controller.go +++ b/test/testutils/controller.go @@ -7,10 +7,16 @@ import ( "log" "os" "path/filepath" + "sync" "testing" "time" + constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" + "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" "github.com/open-policy-agent/gatekeeper/v3/apis" + configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/target" + "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var ( @@ -86,7 +93,7 @@ func DeleteObjectAndConfirm(ctx context.Context, t *testing.T, c client.Client, t.Helper() // Construct a single-use Unstructured to send the Delete request. - toDelete := makeUnstructured(gvk, namespace, name) + toDelete := UnstructuredFor(gvk, namespace, name) err := c.Delete(ctx, toDelete) if apierrors.IsNotFound(err) { return @@ -99,7 +106,7 @@ func DeleteObjectAndConfirm(ctx context.Context, t *testing.T, c client.Client, }, func() error { // Construct a single-use Unstructured to send the Get request. It isn't // safe to reuse Unstructureds for each retry as Get modifies its input. - toGet := makeUnstructured(gvk, namespace, name) + toGet := UnstructuredFor(gvk, namespace, name) key := client.ObjectKey{Namespace: namespace, Name: name} err2 := c.Get(ctx, key, toGet) if apierrors.IsGone(err2) || apierrors.IsNotFound(err2) { @@ -168,12 +175,83 @@ func CreateThenCleanup(ctx context.Context, t *testing.T, c client.Client, obj c t.Cleanup(DeleteObjectAndConfirm(ctx, t, c, obj)) } -func makeUnstructured(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { - u := &unstructured.Unstructured{ - Object: make(map[string]interface{}), +func SetupDataClient(t *testing.T) *constraintclient.Client { + driver, err := rego.New(rego.Tracing(false)) + if err != nil { + t.Fatalf("setting up Driver: %v", err) + } + + client, err := constraintclient.NewClient(constraintclient.Targets(&target.K8sValidationTarget{}), constraintclient.Driver(driver)) + if err != nil { + t.Fatalf("setting up constraint framework client: %v", err) + } + return client +} + +// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and +// writes the request to requests after Reconcile is finished. +func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, *sync.Map) { + var requests sync.Map + fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + result, err := inner.Reconcile(ctx, req) + requests.Store(req, struct{}{}) + return result, err + }) + return fn, &requests +} + +// ConfigFor returns a config resource that watches the requested set of resources. +func ConfigFor(kinds []schema.GroupVersionKind) *configv1alpha1.Config { + entries := make([]configv1alpha1.SyncOnlyEntry, len(kinds)) + for i := range kinds { + entries[i].Group = kinds[i].Group + entries[i].Version = kinds[i].Version + entries[i].Kind = kinds[i].Kind + } + + return &configv1alpha1.Config{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configv1alpha1.GroupVersion.String(), + Kind: "Config", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "config", + Namespace: "gatekeeper-system", + }, + Spec: configv1alpha1.ConfigSpec{ + Sync: configv1alpha1.Sync{ + SyncOnly: entries, + }, + Match: []configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }, + }, } +} + +func UnstructuredFor(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} u.SetGroupVersionKind(gvk) - u.SetNamespace(namespace) u.SetName(name) + if namespace == "" { + u.SetNamespace("default") + } else { + u.SetNamespace(namespace) + } + + if gvk.Kind == "Pod" { + u.Object["spec"] = map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "foo-container", + "image": "foo-image", + }, + }, + } + } + return u } From 3d43814ddf9ba9be414f46de27e20c94d99f1e8e Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 3 Oct 2023 00:11:44 +0000 Subject: [PATCH 02/42] add syncset readiness, observe config Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- apis/syncset/v1alpha1/syncset_types.go | 9 + pkg/controller/config/config_controller.go | 2 + .../config/config_controller_suite_test.go | 15 -- .../config/config_controller_test.go | 2 +- pkg/controller/syncset/syncset_controller.go | 2 +- pkg/readiness/helpers_test.go | 1 + pkg/readiness/object_tracker.go | 23 ++- pkg/readiness/ready_tracker.go | 192 +++++++++++++----- pkg/readiness/ready_tracker_test.go | 109 ++++++---- pkg/readiness/testdata/99-syncset.yaml | 10 + .../bad-gvk}/99-config.yaml | 0 .../testdata/config/empty-sync/99-config.yaml | 9 + .../config/repeating-gvk/99-config.yaml | 14 ++ .../testdata/syncset/99-syncset-1.yaml | 12 ++ .../testdata/syncset/99-syncset-2.yaml | 12 ++ .../testdata/syncset/99-syncset-3.yaml | 12 ++ 16 files changed, 317 insertions(+), 107 deletions(-) create mode 100644 pkg/readiness/testdata/99-syncset.yaml rename pkg/readiness/testdata/{bogus-config => config/bad-gvk}/99-config.yaml (100%) create mode 100644 pkg/readiness/testdata/config/empty-sync/99-config.yaml create mode 100644 pkg/readiness/testdata/config/repeating-gvk/99-config.yaml create mode 100644 pkg/readiness/testdata/syncset/99-syncset-1.yaml create mode 100644 pkg/readiness/testdata/syncset/99-syncset-2.yaml create mode 100644 pkg/readiness/testdata/syncset/99-syncset-3.yaml diff --git a/apis/syncset/v1alpha1/syncset_types.go b/apis/syncset/v1alpha1/syncset_types.go index d03b53c5bff..a542e4691c4 100644 --- a/apis/syncset/v1alpha1/syncset_types.go +++ b/apis/syncset/v1alpha1/syncset_types.go @@ -2,6 +2,7 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" ) type SyncSetSpec struct { @@ -14,6 +15,14 @@ type GVKEntry struct { Kind string `json:"kind,omitempty"` } +func (e *GVKEntry) ToGroupVersionKind() schema.GroupVersionKind { + return schema.GroupVersionKind{ + Group: e.Group, + Version: e.Version, + Kind: e.Kind, + } +} + // +kubebuilder:resource:scope=Cluster // +kubebuilder:object:root=true diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 59c2d9db3a4..bf2c75e571f 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -177,6 +177,8 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque // sync anything gvksToSync := []schema.GroupVersionKind{} if exists && instance.GetDeletionTimestamp().IsZero() { + r.tracker.For(instance.GroupVersionKind()).Observe(instance) + for _, entry := range instance.Spec.Sync.SyncOnly { gvk := schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind} gvksToSync = append(gvksToSync, gvk) diff --git a/pkg/controller/config/config_controller_suite_test.go b/pkg/controller/config/config_controller_suite_test.go index 3932f427015..de77416982f 100644 --- a/pkg/controller/config/config_controller_suite_test.go +++ b/pkg/controller/config/config_controller_suite_test.go @@ -16,11 +16,9 @@ limitations under the License. package config import ( - "context" stdlog "log" "os" "path/filepath" - "sync" "testing" "github.com/open-policy-agent/gatekeeper/v3/apis" @@ -28,7 +26,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var cfg *rest.Config @@ -60,15 +57,3 @@ func TestMain(m *testing.M) { } os.Exit(code) } - -// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and -// writes the request to requests after Reconcile is finished. -func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, *sync.Map) { - var requests sync.Map - fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - result, err := inner.Reconcile(ctx, req) - requests.Store(req, struct{}{}) - return result, err - }) - return fn, &requests -} diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index ce6538ac2dc..2e08ebf7f75 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -155,7 +155,7 @@ func TestReconcile(t *testing.T) { require.NoError(t, err) // Wrap the Controller Reconcile function so it writes each request to a map when it is finished reconciling. - recFn, requests := SetupTestReconcile(rec) + recFn, requests := testutils.SetupTestReconcile(rec) require.NoError(t, add(mgr, recFn)) testutils.StartManager(ctx, t, mgr) diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index 9eca69f7684..f3859445d1b 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -144,12 +144,12 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } } - syncsetTr.Observe(instance) } gvks := make([]schema.GroupVersionKind, 0) if exists && instance.GetDeletionTimestamp().IsZero() { log.Info("handling SyncSet update", "instance", instance) + syncsetTr.Observe(instance) for _, entry := range instance.Spec.GVKs { gvks = append(gvks, schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind}) diff --git a/pkg/readiness/helpers_test.go b/pkg/readiness/helpers_test.go index ebdf4a4864b..3c723e3c72b 100644 --- a/pkg/readiness/helpers_test.go +++ b/pkg/readiness/helpers_test.go @@ -29,6 +29,7 @@ import ( ) // Applies fixture YAMLs directly under the provided path in alpha-sorted order. +// Does not crawl the directory, instead it only looks at the files present at path. func applyFixtures(path string) error { files, err := os.ReadDir(path) if err != nil { diff --git a/pkg/readiness/object_tracker.go b/pkg/readiness/object_tracker.go index 674824106a5..ee09f5f4ff2 100644 --- a/pkg/readiness/object_tracker.go +++ b/pkg/readiness/object_tracker.go @@ -22,7 +22,8 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" - "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" + configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + syncset1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -242,12 +243,16 @@ func (t *objectTracker) Observe(o runtime.Object) { _, wasExpecting := t.expect[k] switch { case wasExpecting: + log.V(1).Info("[readiness] observing expectation", "gvk", o.GetObjectKind().GroupVersionKind()) + // Satisfy existing expectation delete(t.seen, k) delete(t.expect, k) t.satisfied[k] = struct{}{} return case !wasExpecting && t.populated: + log.V(1).Info("[readiness] not expecting anymore", "gvk", o.GetObjectKind().GroupVersionKind()) + // Not expecting and no longer accepting expectations. // No need to track. delete(t.seen, k) @@ -257,7 +262,7 @@ func (t *objectTracker) Observe(o runtime.Object) { // Track for future expectation. t.seen[k] = struct{}{} - log.V(logging.DebugLevel).Info("[readiness] observed data", "gvk", o.GetObjectKind().GroupVersionKind()) + log.V(1).Info("[readiness] observed data", "gvk", o.GetObjectKind().GroupVersionKind()) } func (t *objectTracker) Populated() bool { @@ -300,7 +305,7 @@ func (t *objectTracker) Satisfied() bool { if !needMutate { // Read lock to prevent concurrent read/write while logging readiness state. t.mu.RLock() - log.V(1).Info("readiness state", "gvk", t.gvk, "satisfied", fmt.Sprintf("%d/%d", len(t.satisfied), len(t.expect)+len(t.satisfied))) + log.V(1).Info("readiness state unsatisfied", "gvk", t.gvk, "satisfied", fmt.Sprintf("%d/%d", len(t.satisfied), len(t.expect)+len(t.satisfied))) t.mu.RUnlock() return false } @@ -396,6 +401,18 @@ func objKeyFromObject(obj runtime.Object) (objKey, error) { Version: v1beta1.SchemeGroupVersion.Version, Kind: v.Spec.CRD.Spec.Names.Kind, } + case *configv1alpha1.Config: + gvk = schema.GroupVersionKind{ + Group: configv1alpha1.GroupVersion.Group, + Version: configv1alpha1.GroupVersion.Version, + Kind: "Config", + } + case *syncset1alpha1.SyncSet: + gvk = schema.GroupVersionKind{ + Group: syncset1alpha1.GroupVersion.Group, + Version: syncset1alpha1.GroupVersion.Version, + Kind: "SyncSet", + } case *unstructured.Unstructured: ugvk := obj.GetObjectKind().GroupVersionKind() if ugvk.GroupVersion() == v1beta1.SchemeGroupVersion && ugvk.Kind == "ConstraintTemplate" { diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index a2eeab4658b..a5adcae8bf6 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -29,6 +29,7 @@ import ( expansionv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/expansion/v1alpha1" mutationv1 "github.com/open-policy-agent/gatekeeper/v3/apis/mutations/v1" mutationsv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/mutations/v1alpha1" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" "github.com/open-policy-agent/gatekeeper/v3/pkg/operations" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" @@ -63,6 +64,7 @@ type Tracker struct { templates *objectTracker config *objectTracker + syncsets *objectTracker assignMetadata *objectTracker assign *objectTracker modifySet *objectTracker @@ -90,6 +92,7 @@ func newTracker(lister Lister, mutationEnabled, externalDataEnabled, expansionEn lister: lister, templates: newObjTracker(v1beta1.SchemeGroupVersion.WithKind("ConstraintTemplate"), fn), config: newObjTracker(configv1alpha1.GroupVersion.WithKind("Config"), fn), + syncsets: newObjTracker(syncsetv1alpha1.GroupVersion.WithKind("SyncSet"), fn), constraints: newTrackerMap(fn), data: newTrackerMap(fn), ready: make(chan struct{}), @@ -136,6 +139,8 @@ func (t *Tracker) For(gvk schema.GroupVersionKind) Expectations { return t.templates } return noopExpectations{} + case gvk.Group == syncsetv1alpha1.GroupVersion.Group && gvk.Kind == "SyncSet": + return t.syncsets case gvk.Group == configv1alpha1.GroupVersion.Group && gvk.Kind == "Config": return t.config case gvk.Group == externaldatav1beta1.SchemeGroupVersion.Group && gvk.Kind == "Provider": @@ -180,13 +185,20 @@ func (t *Tracker) For(gvk schema.GroupVersionKind) Expectations { func (t *Tracker) ForData(gvk schema.GroupVersionKind) Expectations { // Avoid new data trackers after data expectations have been fully populated. // Race is ok here - extra trackers will only consume some unneeded memory. - if t.config.Populated() && !t.data.Has(gvk) { + if t.DataPopulated() && !t.data.Has(gvk) { // Return throw-away tracker instead. return noopExpectations{} } return t.data.Get(gvk) } +func (t *Tracker) DataGVKs() []schema.GroupVersionKind { + t.mu.RLock() + defer t.mu.RUnlock() + + return t.data.Keys() +} + func (t *Tracker) templateCleanup(ct *templates.ConstraintTemplate) { gvk := constraintGVK(ct) t.constraints.Remove(gvk) @@ -265,13 +277,19 @@ func (t *Tracker) Satisfied() bool { log.V(1).Info("all expectations satisfied", "tracker", "constraints") if !t.config.Satisfied() { + log.V(1).Info("expectations unsatisfied", "tracker", "config") + return false + } + + if !t.syncsets.Satisfied() { + log.V(1).Info("expectations unsatisfied", "tracker", "syncset") return false } if operations.HasValidationOperations() { - configKinds := t.config.kinds() - for _, gvk := range configKinds { + for _, gvk := range t.data.Keys() { if !t.data.Get(gvk).Satisfied() { + log.V(1).Info("expectations unsatisfied", "tracker", "data", "gvk", gvk) return false } } @@ -322,10 +340,10 @@ func (t *Tracker) Run(ctx context.Context) error { grp.Go(func() error { return t.trackConstraintTemplates(gctx) }) + grp.Go(func() error { + return t.trackSyncSources(gctx) + }) } - grp.Go(func() error { - return t.trackConfig(gctx) - }) grp.Go(func() error { t.statsPrinter(ctx) return nil @@ -379,18 +397,32 @@ func (t *Tracker) Populated() bool { if operations.HasValidationOperations() { validationPopulated = t.templates.Populated() && t.constraints.Populated() && t.data.Populated() } - return validationPopulated && t.config.Populated() && mutationPopulated && externalDataProviderPopulated + return validationPopulated && t.config.Populated() && mutationPopulated && externalDataProviderPopulated && t.syncsets.Populated() +} + +func (t *Tracker) DataPopulated() bool { + dataPopulated := true + if operations.HasValidationOperations() { + dataPopulated = t.data.Populated() + } + + return dataPopulated && t.config.Populated() && t.syncsets.Populated() } // collectForObjectTracker identifies objects that are unsatisfied for the provided // `es`, which must be an objectTracker, and removes those expectations. -func (t *Tracker) collectForObjectTracker(ctx context.Context, es Expectations, cleanup func(schema.GroupVersionKind)) error { +func (t *Tracker) collectForObjectTracker(ctx context.Context, es Expectations, cleanup func(schema.GroupVersionKind), trackerName string) error { if es == nil { return fmt.Errorf("nil Expectations provided to collectForObjectTracker") } - if !es.Populated() || es.Satisfied() { - log.V(1).Info("Expectations unpopulated or already satisfied, skipping collection") + if !es.Populated() { + log.V(1).Info("Expectations unpopulated, skipping collection", "tracker", trackerName) + return nil + } + + if es.Satisfied() { + log.V(1).Info("Expectations already satisfied, skipping collection", "tracker", trackerName) return nil } @@ -433,6 +465,8 @@ func (t *Tracker) collectForObjectTracker(ctx context.Context, es Expectations, u.SetName(k.namespacedName.Name) u.SetNamespace(k.namespacedName.Namespace) u.SetGroupVersionKind(k.gvk) + + log.V(1).Info("canceling expectations", "name", u.GetName(), "namespace", u.GetNamespace(), "gvk", u.GroupVersionKind(), "tracker", trackerName) ot.CancelExpect(u) if cleanup != nil { cleanup(gvk) @@ -453,25 +487,28 @@ func (t *Tracker) collectInvalidExpectations(ctx context.Context) { t.constraints.Remove(gvk) t.constraintTrackers.Cancel(gvk.String()) } - err := t.collectForObjectTracker(ctx, tt, cleanupTemplate) + err := t.collectForObjectTracker(ctx, tt, cleanupTemplate, "ConstraintTemplate") if err != nil { log.Error(err, "while collecting for the ConstraintTemplate tracker") } ct := t.config - cleanupData := func(gvk schema.GroupVersionKind) { - t.data.Remove(gvk) - } - err = t.collectForObjectTracker(ctx, ct, cleanupData) + err = t.collectForObjectTracker(ctx, ct, nil, "Config") if err != nil { log.Error(err, "while collecting for the Config tracker") } + sst := t.syncsets + err = t.collectForObjectTracker(ctx, sst, nil, "SyncSet") + if err != nil { + log.Error(err, "while collecting for the SyncSet tracker") + } + // collect deleted but expected constraints for _, gvk := range t.constraints.Keys() { // retrieve the expectations for this key es := t.constraints.Get(gvk) - err = t.collectForObjectTracker(ctx, es, nil) + err = t.collectForObjectTracker(ctx, es, nil, fmt.Sprintf("%s/%s", "Constraints", gvk)) if err != nil { log.Error(err, "while collecting for the Constraint type", "gvk", gvk) continue @@ -482,7 +519,7 @@ func (t *Tracker) collectInvalidExpectations(ctx context.Context) { for _, gvk := range t.data.Keys() { // retrieve the expectations for this key es := t.data.Get(gvk) - err = t.collectForObjectTracker(ctx, es, nil) + err = t.collectForObjectTracker(ctx, es, nil, fmt.Sprintf("%s/%s", "Data", gvk)) if err != nil { log.Error(err, "while collecting for the Data type", "gvk", gvk) continue @@ -689,58 +726,87 @@ func (t *Tracker) trackConstraintTemplates(ctx context.Context) error { return nil } -// trackConfig sets expectations for cached data as specified by the singleton Config resource. -// Fails-open if the Config resource cannot be fetched or does not exist. -func (t *Tracker) trackConfig(ctx context.Context) error { +// trackSyncSources sets expectations for cached data as specified by the singleton Config resource. +// and any SyncSet resources present on the cluster. +// Works best effort and fails-open if the a resource cannot be fetched or does not exist. +func (t *Tracker) trackSyncSources(ctx context.Context) error { var wg sync.WaitGroup defer func() { - defer t.config.ExpectationsDone() + t.config.ExpectationsDone() log.V(1).Info("config expectations populated") + t.syncsets.ExpectationsDone() + log.V(1).Info("syncset expectations populated") + wg.Wait() }() + handled := make(map[schema.GroupVersionKind]struct{}) + cfg, err := t.getConfigResource(ctx) if err != nil { - return fmt.Errorf("fetching config resource: %w", err) + log.Error(err, "fetching config resource") } if cfg == nil { log.Info("config resource not found - skipping for readiness") - return nil - } - if !cfg.GetDeletionTimestamp().IsZero() { - log.Info("config resource is being deleted - skipping for readiness") - return nil + } else { + if !cfg.GetDeletionTimestamp().IsZero() { + log.Info("config resource is being deleted - skipping for readiness") + } else { + t.config.Expect(cfg) + log.V(1).Info("setting expectations for config", "configCount", 1) + + for _, entry := range cfg.Spec.Sync.SyncOnly { + handled[schema.GroupVersionKind{ + Group: entry.Group, + Version: entry.Version, + Kind: entry.Kind, + }] = struct{}{} + } + } } - if operations.HasValidationOperations() { - // Expect the resource kinds specified in the Config. - // We will fail-open (resolve expectations) for GVKs - // that are unregistered. - for _, entry := range cfg.Spec.Sync.SyncOnly { - gvk := schema.GroupVersionKind{ - Group: entry.Group, - Version: entry.Version, - Kind: entry.Kind, - } - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - t.config.Expect(u) - t.config.Observe(u) // we only care about the gvk entry in kinds() - - // Set expectations for individual cached resources - dt := t.ForData(gvk) - wg.Add(1) - go func() { - defer wg.Done() - err := t.trackData(ctx, gvk, dt) - if err != nil { - log.Error(err, "aborted trackData", "gvk", gvk) + syncsets := &syncsetv1alpha1.SyncSetList{} + lister := retryLister(t.lister, retryAll) + if err := lister.List(ctx, syncsets); err != nil { + log.Error(err, "listing syncsets") + } else { + log.V(1).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) + + for i := range syncsets.Items { + syncset := syncsets.Items[i] + + t.syncsets.Expect(&syncset) + log.V(1).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) + + for i := range syncset.Spec.GVKs { + gvk := syncset.Spec.GVKs[i].ToGroupVersionKind() + if _, ok := handled[gvk]; ok { + log.Info("duplicate GVK to sync", "gvk", gvk) } - }() + + handled[gvk] = struct{}{} + } } } + // Expect the resource kinds specified in the Config resource and all SyncSet resources. + // We will fail-open (resolve expectations) for GVKs that are unregistered. + for gvk := range handled { + g := gvk + + // Set expectations for individual cached resources + dt := t.ForData(g) + wg.Add(1) + go func() { + defer wg.Done() + err := t.trackData(ctx, g, dt) + if err != nil { + log.Error(err, "aborted trackData", "gvk", g) + } + }() + } + return nil } @@ -878,7 +944,7 @@ func (t *Tracker) statsPrinter(ctx context.Context) { } } - for _, gvk := range t.config.kinds() { + for _, gvk := range t.data.Keys() { if t.data.Get(gvk).Satisfied() { continue } @@ -897,6 +963,9 @@ func (t *Tracker) statsPrinter(ctx context.Context) { log.Info("unsatisfied data", "name", u.namespacedName, "gvk", u.gvk) } } + + logUnsatisfiedSyncSet(t) + logUnsatisfiedConfig(t) } if t.mutationEnabled { logUnsatisfiedAssignMetadata(t) @@ -913,6 +982,25 @@ func (t *Tracker) statsPrinter(ctx context.Context) { } } +func logUnsatisfiedSyncSet(t *Tracker) { + if unsat := t.syncsets.unsatisfied(); len(unsat) > 0 { + log.Info("--- Begin unsatisfied syncsets ---", "populated", t.syncsets.Populated(), "count", len(unsat)) + + for _, k := range unsat { + log.Info("unsatisfied SyncSet", "name", k.namespacedName) + } + log.Info("--- End unsatisfied syncsets ---") + } +} + +func logUnsatisfiedConfig(t *Tracker) { + if unsat := t.config.unsatisfied(); len(unsat) > 0 { + log.Info("--- Begin unsatisfied config ---", "populated", t.config.Populated(), "count", len(unsat)) + log.Info("unsatisfied Config", "name", keys.Config) + log.Info("--- End unsatisfied config ---") + } +} + func logUnsatisfiedAssignMetadata(t *Tracker) { for _, amKey := range t.assignMetadata.unsatisfied() { log.Info("unsatisfied AssignMetadata", "name", amKey.namespacedName) diff --git a/pkg/readiness/ready_tracker_test.go b/pkg/readiness/ready_tracker_test.go index a31e47d2f74..440f063eccb 100644 --- a/pkg/readiness/ready_tracker_test.go +++ b/pkg/readiness/ready_tracker_test.go @@ -30,6 +30,8 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" frameworksexternaldata "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" + configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" @@ -44,6 +46,8 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -57,6 +61,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics" ) +const ( + ttimeout = 20 * time.Second + ttick = 1 * time.Second +) + // setupManager sets up a controller-runtime manager with registered watch manager. func setupManager(t *testing.T) (manager.Manager, *watch.Manager) { t.Helper() @@ -543,41 +552,63 @@ func Test_Tracker(t *testing.T) { } } -// Verifies that a Config resource referencing bogus (unregistered) GVKs will not halt readiness. -func Test_Tracker_UnregisteredCachedData(t *testing.T) { - g := gomega.NewWithT(t) - - testutils.Setenv(t, "POD_NAME", "no-pod") - - // Apply fixtures *before* the controllers are setup. - err := applyFixtures("testdata") - if err != nil { - t.Fatalf("applying fixtures: %v", err) - } - - // Apply config resource with bogus GVK reference - err = applyFixtures("testdata/bogus-config") - if err != nil { - t.Fatalf("applying config: %v", err) - } - - // Wire up the rest. - mgr, wm := setupManager(t) - cfClient := setupDataClient(t) - providerCache := frameworksexternaldata.NewCache() - - if err := setupController(mgr, wm, cfClient, mutation.NewSystem(mutation.SystemOpts{}), nil, providerCache); err != nil { - t.Fatalf("setupControllers: %v", err) +// Verifies additional scenarios to the base "testdata/" fixtures, such as +// invalid Config resources or overlapping SyncSets, which we want to +// make sure the ReadyTracker can handle gracefully. +func Test_Tracker_EdgeCases(t *testing.T) { + tts := []struct { + name string + fixturesPath string + }{ + { + name: "overlapping syncsets", + fixturesPath: "testdata/syncset", + }, + { + // bad gvk in Config doesn't halt readiness + name: "bad gvk", + fixturesPath: "testdata/config/bad-gvk", + }, + { + // repeating gvk in Config doesn't halt readiness + name: "repeating gvk", + fixturesPath: "testdata/config/repeating-gvk", + }, + { + // empty syncOnly still needs reconciling for a Config resources + name: "empty sync only gvk", + fixturesPath: "testdata/config/empty-sync", + }, + } + + for _, tt := range tts { + t.Run(tt.name, func(t *testing.T) { + testutils.Setenv(t, "POD_NAME", "no-pod") + require.NoError(t, applyFixtures("testdata"), "base fixtures config") + require.NoError(t, applyFixtures("testdata/config/bad-gvk"), fmt.Sprintf("test fixtures: %s", tt.fixturesPath)) + + mgr, wm := setupManager(t) + cfClient := testutils.SetupDataClient(t) + providerCache := frameworksexternaldata.NewCache() + + require.NoError(t, setupController(mgr, wm, cfClient, mutation.NewSystem(mutation.SystemOpts{}), nil, providerCache)) + + ctx, cancelFunc := context.WithCancel(context.Background()) + testutils.StartManager(ctx, t, mgr) + + require.Eventually(t, func() bool { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + ready, err := probeIsReady(ctx) + assert.NoError(t, err, "error while waiting for probe to be ready") + + return ready + }, ttimeout, ttick, "tracker not healthy") + + cancelFunc() + }) } - - ctx := context.Background() - testutils.StartManager(ctx, t, mgr) - - g.Eventually(func() (bool, error) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - return probeIsReady(ctx) - }, 20*time.Second, 1*time.Second).Should(gomega.BeTrue()) } // Test_CollectDeleted adds resources and starts the readiness tracker, then @@ -642,6 +673,13 @@ func Test_CollectDeleted(t *testing.T) { if err != nil { t.Fatalf("retrieving ConstraintTemplate GVK: %v", err) } + config := &configv1alpha1.Config{} + configGvk, err := apiutil.GVKForObject(config, mgr.GetScheme()) + require.NoError(t, err) + + syncset := &syncsetv1alpha1.SyncSet{} + syncsetGvk, err := apiutil.GVKForObject(syncset, mgr.GetScheme()) + require.NoError(t, err) // note: state can leak between these test cases because we do not reset the environment // between them to keep the test short. Trackers are mostly independent per GVK. @@ -649,8 +687,9 @@ func Test_CollectDeleted(t *testing.T) { {description: "constraints", gvk: cgvk}, {description: "data (configmaps)", gvk: cmgvk, tracker: &cmtracker}, {description: "templates", gvk: ctgvk}, - // no need to check Config here since it is not actually Expected for readiness // (the objects identified in a Config's syncOnly are Expected, tested in data case above) + {description: "config", gvk: configGvk}, + {description: "syncset", gvk: syncsetGvk}, } for _, tc := range tests { diff --git a/pkg/readiness/testdata/99-syncset.yaml b/pkg/readiness/testdata/99-syncset.yaml new file mode 100644 index 00000000000..aa92d4af0d7 --- /dev/null +++ b/pkg/readiness/testdata/99-syncset.yaml @@ -0,0 +1,10 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-1 + namespace: "gatekeeper-system" +spec: + gvks: + - group: "" + version: "v1" + kind: "ConfigMap" diff --git a/pkg/readiness/testdata/bogus-config/99-config.yaml b/pkg/readiness/testdata/config/bad-gvk/99-config.yaml similarity index 100% rename from pkg/readiness/testdata/bogus-config/99-config.yaml rename to pkg/readiness/testdata/config/bad-gvk/99-config.yaml diff --git a/pkg/readiness/testdata/config/empty-sync/99-config.yaml b/pkg/readiness/testdata/config/empty-sync/99-config.yaml new file mode 100644 index 00000000000..4bcb9908dde --- /dev/null +++ b/pkg/readiness/testdata/config/empty-sync/99-config.yaml @@ -0,0 +1,9 @@ +apiVersion: config.gatekeeper.sh/v1alpha1 +kind: Config +metadata: + name: config + namespace: "gatekeeper-system" +spec: + match: + - excludedNamespaces: ["kube-*", "gatekeeper-system"] + processes: ["*"] diff --git a/pkg/readiness/testdata/config/repeating-gvk/99-config.yaml b/pkg/readiness/testdata/config/repeating-gvk/99-config.yaml new file mode 100644 index 00000000000..19c84034def --- /dev/null +++ b/pkg/readiness/testdata/config/repeating-gvk/99-config.yaml @@ -0,0 +1,14 @@ +apiVersion: config.gatekeeper.sh/v1alpha1 +kind: Config +metadata: + name: config + namespace: "gatekeeper-system" +spec: + sync: + syncOnly: + - group: "" + version: "v1" + kind: "ConfigMap" + - group: "" + version: "v1" + kind: "ConfigMap" diff --git a/pkg/readiness/testdata/syncset/99-syncset-1.yaml b/pkg/readiness/testdata/syncset/99-syncset-1.yaml new file mode 100644 index 00000000000..3f1f8450e3c --- /dev/null +++ b/pkg/readiness/testdata/syncset/99-syncset-1.yaml @@ -0,0 +1,12 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-1 +spec: + gvks: + - group: "" + version: "v1" + kind: "Pod" + - group: "" + version: "v1" + kind: "ConfigMap" diff --git a/pkg/readiness/testdata/syncset/99-syncset-2.yaml b/pkg/readiness/testdata/syncset/99-syncset-2.yaml new file mode 100644 index 00000000000..b18d51248d9 --- /dev/null +++ b/pkg/readiness/testdata/syncset/99-syncset-2.yaml @@ -0,0 +1,12 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-2 +spec: + gvks: + - group: "" + version: "v1" + kind: "Namespace" + - group: "" + version: "v1" + kind: "ConfigMap" diff --git a/pkg/readiness/testdata/syncset/99-syncset-3.yaml b/pkg/readiness/testdata/syncset/99-syncset-3.yaml new file mode 100644 index 00000000000..7587d8ce22a --- /dev/null +++ b/pkg/readiness/testdata/syncset/99-syncset-3.yaml @@ -0,0 +1,12 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-3 +spec: + gvks: + - group: "" + version: "v1" + kind: "Pod" + - group: "" + version: "v1" + kind: "Namespace" From 0dc69bd440fb11c16b25f4e908a3f0fd85be9503 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 3 Oct 2023 00:17:20 +0000 Subject: [PATCH 03/42] reconcile data expectations Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- main.go | 4 + pkg/cachemanager/cachemanager.go | 30 ++- pkg/expectationsmgr/expectationsmgr.go | 58 ++++++ pkg/expectationsmgr/expectationsmgr_test.go | 183 ++++++++++++++++++ .../00-namespace.yaml | 4 + .../20-syncset-1.yaml | 9 + .../syncsets-config-disjoint/21-config.yaml | 14 ++ .../syncsets-overlapping/20-syncset-1.yaml | 12 ++ .../syncsets-overlapping/20-syncset-2.yaml | 12 ++ .../syncsets-overlapping/20-syncset-3.yaml | 12 ++ .../testdata/syncsets-resources/00-gk-ns.yaml | 4 + .../syncsets-resources/11-config.yaml | 9 + .../syncsets-resources/15-configmap-1.yaml | 7 + .../syncsets-resources/20-syncset-1.yaml | 9 + .../syncsets-resources/20-syncset-2.yaml | 9 + test/testutils/applier.go | 67 +++++++ 16 files changed, 434 insertions(+), 9 deletions(-) create mode 100644 pkg/expectationsmgr/expectationsmgr.go create mode 100644 pkg/expectationsmgr/expectationsmgr_test.go create mode 100644 pkg/expectationsmgr/testdata/syncsets-config-disjoint/00-namespace.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-config-disjoint/20-syncset-1.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-config-disjoint/21-config.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-1.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-2.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-3.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-resources/00-gk-ns.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-resources/11-config.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-resources/15-configmap-1.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-1.yaml create mode 100644 pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-2.yaml create mode 100644 test/testutils/applier.go diff --git a/main.go b/main.go index 9bc7f04ff26..216afb6b5a6 100644 --- a/main.go +++ b/main.go @@ -47,6 +47,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/controller" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" + "github.com/open-policy-agent/gatekeeper/v3/pkg/expectationsmgr" "github.com/open-policy-agent/gatekeeper/v3/pkg/externaldata" "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" @@ -476,6 +477,9 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle return err } + em := expectationsmgr.NewExpecationsManager(cm, tracker) + go em.Run(ctx) + opts := controller.Dependencies{ CFClient: client, WatchManger: wm, diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 00b1691d26a..bc2cc1c0919 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -59,6 +59,8 @@ type CacheManager struct { registrar *watch.Registrar backgroundManagementTicker time.Ticker reader client.Reader + + started bool } // CFDataClient is an interface for caching data. @@ -104,10 +106,21 @@ func NewCacheManager(config *Config) (*CacheManager, error) { func (c *CacheManager) Start(ctx context.Context) error { go c.manageCache(ctx) + c.mu.Lock() + c.started = true + c.mu.Unlock() + <-ctx.Done() return nil } +func (c *CacheManager) Started() bool { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.started +} + // UpsertSource adjusts the watched set of gvks according to the newGVKs passed in // for a given sourceKey. Callers are responsible for retrying on error. func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { @@ -142,8 +155,7 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet.Add(c.gvksToSync.GVKs()...) diff := c.watchedSet.Difference(newWatchSet) - c.removeStaleExpectations(diff) - + // any resulting stale data expectations are handled async by the ExpectationsMgr. c.gvksToDeleteFromCache.AddSet(diff) var innerError error @@ -158,13 +170,6 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return innerError } -// removeStaleExpectations stops tracking data for any resources that are no longer watched. -func (c *CacheManager) removeStaleExpectations(stale *watch.Set) { - for _, gvk := range stale.Items() { - c.tracker.CancelData(gvk) - } -} - // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() @@ -208,6 +213,13 @@ func (c *CacheManager) DoForEach(fn func(gvk schema.GroupVersionKind) error) err return err } +func (c *CacheManager) WatchedGVKs() []schema.GroupVersionKind { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.watchedSet.Items() +} + func (c *CacheManager) watchesGVK(gvk schema.GroupVersionKind) bool { c.mu.RLock() defer c.mu.RUnlock() diff --git a/pkg/expectationsmgr/expectationsmgr.go b/pkg/expectationsmgr/expectationsmgr.go new file mode 100644 index 00000000000..3bbe22c1b21 --- /dev/null +++ b/pkg/expectationsmgr/expectationsmgr.go @@ -0,0 +1,58 @@ +package expectationsmgr + +import ( + "context" + "time" + + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" + "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" +) + +const tickDuration = 3 * time.Second + +// ExpectationsMgr fires after data expectations have been populated in the ready Tracker and runs +// until the Tracker is satisfeid or the process is exiting. It removes Data expectations for any +// GVKs that are expected in the Tracker but not watched by the CacheManager. +type ExpectationsMgr struct { + cacheMgr *cachemanager.CacheManager + tracker *readiness.Tracker +} + +func NewExpecationsManager(cm *cachemanager.CacheManager, rt *readiness.Tracker) *ExpectationsMgr { + return &ExpectationsMgr{ + cacheMgr: cm, + tracker: rt, + } +} + +func (e *ExpectationsMgr) Run(ctx context.Context) { + ticker := time.NewTicker(tickDuration) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if e.tracker.Satisfied() { + // we're done, there's no need to + // further manage the data sync expectations. + return + } + if !(e.tracker.DataPopulated() && e.cacheMgr.Started()) { + // we have to wait on data expectations to be populated + // and for the cachemanager to have been started by the + // controller manager. + break + } + + watchedGVKs := watch.NewSet() + watchedGVKs.Add(e.cacheMgr.WatchedGVKs()...) + expectedGVKs := watch.NewSet() + expectedGVKs.Add(e.tracker.DataGVKs()...) + + for _, gvk := range expectedGVKs.Difference(watchedGVKs).Items() { + e.tracker.CancelData(gvk) + } + } + } +} diff --git a/pkg/expectationsmgr/expectationsmgr_test.go b/pkg/expectationsmgr/expectationsmgr_test.go new file mode 100644 index 00000000000..b2c737cdc9b --- /dev/null +++ b/pkg/expectationsmgr/expectationsmgr_test.go @@ -0,0 +1,183 @@ +package expectationsmgr + +import ( + "context" + "fmt" + "testing" + "time" + + frameworksexternaldata "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" + configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" + "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" + "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" + testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" + "github.com/open-policy-agent/gatekeeper/v3/test/testutils" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +const ( + timeout = 10 * time.Second + tick = 1 * time.Second +) + +var cfg *rest.Config + +var ( + syncsetGVK = schema.GroupVersionKind{ + Group: syncsetv1alpha1.GroupVersion.Group, + Version: syncsetv1alpha1.GroupVersion.Version, + Kind: "SyncSet", + } + configGVK = configv1alpha1.GroupVersion.WithKind("Config") +) + +func TestMain(m *testing.M) { + testutils.StartControlPlane(m, &cfg, 2) +} + +func setupTest(ctx context.Context, t *testing.T, startControllers bool) (*ExpectationsMgr, client.Client) { + t.Helper() + + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + + tracker, err := readiness.SetupTrackerNoReadyz(mgr, false, false, false) + require.NoError(t, err, "setting up tracker") + + events := make(chan event.GenericEvent, 1024) + reg, err := wm.NewRegistrar( + cachemanager.RegistrarName, + events, + ) + require.NoError(t, err, "creating registrar") + + cfClient := &fakes.FakeCfClient{} + config := &cachemanager.Config{ + CfClient: cfClient, + SyncMetricsCache: syncutil.NewMetricsCache(), + Tracker: tracker, + ProcessExcluder: process.Get(), + Registrar: reg, + Reader: c, + GVKAggregator: aggregator.NewGVKAggregator(), + } + require.NoError(t, err, "creating registrar") + cm, err := cachemanager.NewCacheManager(config) + require.NoError(t, err, "creating cachemanager") + + if !startControllers { + // need to start the cachemanager if controllers are not started + // since the cachemanager is started in the controllers code. + require.NoError(t, mgr.Add(cm), "adding cachemanager as a runnable") + } else { + sw := watch.NewSwitch() + mutationSystem := mutation.NewSystem(mutation.SystemOpts{}) + + frameworksexternaldata.NewCache() + opts := controller.Dependencies{ + CFClient: testutils.SetupDataClient(t), + WatchManger: wm, + ControllerSwitch: sw, + Tracker: tracker, + ProcessExcluder: process.Get(), + MutationSystem: mutationSystem, + ExpansionSystem: expansion.NewSystem(mutationSystem), + ProviderCache: frameworksexternaldata.NewCache(), + CacheMgr: cm, + SyncEventsCh: events, + } + require.NoError(t, controller.AddToManager(mgr, &opts), "registering controllers") + } + + testutils.StartManager(ctx, t, mgr) + + return &ExpectationsMgr{ + cacheMgr: cm, + tracker: tracker, + }, c +} + +// Test_ExpectationsMgr_DeletedSyncSets tests scenarios in which SyncSet and Config resources +// get deleted after tracker expectations have been populated and we need to reconcile +// the GVKs that are in the data client (via the cachemaanger) and the GVKs that are expected +// by the Tracker. +func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { + tts := []struct { + name string + fixturesPath string + syncsetsToDelete []string + deleteConfig string + startControllers bool + }{ + { + name: "delete all syncsets", + fixturesPath: "testdata/syncsets-overlapping", + syncsetsToDelete: []string{"syncset-1", "syncset-2", "syncset-3"}, + }, + { + name: "delete syncs and configs", + fixturesPath: "testdata/syncsets-config-disjoint", + syncsetsToDelete: []string{"syncset-1"}, + deleteConfig: "config", + }, + { + name: "delete one syncset", + fixturesPath: "testdata/syncsets-resources", + syncsetsToDelete: []string{"syncset-2"}, + startControllers: true, + }, + } + + for _, tt := range tts { + t.Run(tt.name, func(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) + + require.NoError(t, testutils.ApplyFixtures(tt.fixturesPath, cfg), "applying base fixtures") + + em, c := setupTest(ctx, t, tt.startControllers) + go em.Run(ctx) + + // we have to wait on the Tracker to Populate in order to not + // have the Deletes below race with the population of expectations. + require.Eventually(t, func() bool { + return em.tracker.Populated() + }, timeout, tick, "waiting on tracker to populate") + + for _, name := range tt.syncsetsToDelete { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(syncsetGVK) + u.SetName(name) + + require.NoError(t, c.Delete(ctx, u), fmt.Sprintf("deleting syncset %s", name)) + } + if tt.deleteConfig != "" { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(configGVK) + u.SetNamespace("gatekeeper-system") + u.SetName(tt.deleteConfig) + + require.NoError(t, c.Delete(ctx, u), fmt.Sprintf("deleting config %s", tt.deleteConfig)) + } + + require.Eventually(t, func() bool { + return em.tracker.Satisfied() + }, timeout, tick, "waiting on tracker to get satisfied") + + cancelFunc() + }) + } +} diff --git a/pkg/expectationsmgr/testdata/syncsets-config-disjoint/00-namespace.yaml b/pkg/expectationsmgr/testdata/syncsets-config-disjoint/00-namespace.yaml new file mode 100644 index 00000000000..52a227623e5 --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-config-disjoint/00-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: "gatekeeper-system" diff --git a/pkg/expectationsmgr/testdata/syncsets-config-disjoint/20-syncset-1.yaml b/pkg/expectationsmgr/testdata/syncsets-config-disjoint/20-syncset-1.yaml new file mode 100644 index 00000000000..530ac2d2130 --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-config-disjoint/20-syncset-1.yaml @@ -0,0 +1,9 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-1 +spec: + gvks: + - group: "" + version: "v1" + kind: "Pod" diff --git a/pkg/expectationsmgr/testdata/syncsets-config-disjoint/21-config.yaml b/pkg/expectationsmgr/testdata/syncsets-config-disjoint/21-config.yaml new file mode 100644 index 00000000000..b5fd7165ef4 --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-config-disjoint/21-config.yaml @@ -0,0 +1,14 @@ +apiVersion: config.gatekeeper.sh/v1alpha1 +kind: Config +metadata: + name: config + namespace: "gatekeeper-system" +spec: + sync: + syncOnly: + - group: "" + version: "v1" + kind: "Namespace" + - group: "" + version: "v1" + kind: "ConfigMap" diff --git a/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-1.yaml b/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-1.yaml new file mode 100644 index 00000000000..3f1f8450e3c --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-1.yaml @@ -0,0 +1,12 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-1 +spec: + gvks: + - group: "" + version: "v1" + kind: "Pod" + - group: "" + version: "v1" + kind: "ConfigMap" diff --git a/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-2.yaml b/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-2.yaml new file mode 100644 index 00000000000..b18d51248d9 --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-2.yaml @@ -0,0 +1,12 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-2 +spec: + gvks: + - group: "" + version: "v1" + kind: "Namespace" + - group: "" + version: "v1" + kind: "ConfigMap" diff --git a/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-3.yaml b/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-3.yaml new file mode 100644 index 00000000000..7587d8ce22a --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-3.yaml @@ -0,0 +1,12 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-3 +spec: + gvks: + - group: "" + version: "v1" + kind: "Pod" + - group: "" + version: "v1" + kind: "Namespace" diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/00-gk-ns.yaml b/pkg/expectationsmgr/testdata/syncsets-resources/00-gk-ns.yaml new file mode 100644 index 00000000000..52a227623e5 --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-resources/00-gk-ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: "gatekeeper-system" diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/11-config.yaml b/pkg/expectationsmgr/testdata/syncsets-resources/11-config.yaml new file mode 100644 index 00000000000..357b9c6a870 --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-resources/11-config.yaml @@ -0,0 +1,9 @@ +apiVersion: config.gatekeeper.sh/v1alpha1 +kind: Config +metadata: + name: config + namespace: "gatekeeper-system" +spec: + match: + - excludedNamespaces: ["kube-*"] + processes: ["*"] diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/15-configmap-1.yaml b/pkg/expectationsmgr/testdata/syncsets-resources/15-configmap-1.yaml new file mode 100644 index 00000000000..25a3a17b343 --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-resources/15-configmap-1.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "some-config-map" + namespace: "default" +data: + foo: bar diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-1.yaml b/pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-1.yaml new file mode 100644 index 00000000000..60af509f63c --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-1.yaml @@ -0,0 +1,9 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-1 +spec: + gvks: + - group: "" + version: "v1" + kind: "ConfigMap" diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-2.yaml b/pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-2.yaml new file mode 100644 index 00000000000..b955227a477 --- /dev/null +++ b/pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-2.yaml @@ -0,0 +1,9 @@ +apiVersion: syncset.gatekeeper.sh/v1alpha1 +kind: SyncSet +metadata: + name: syncset-2 +spec: + gvks: + - group: "" + version: "v1" + kind: "UnknownPod" diff --git a/test/testutils/applier.go b/test/testutils/applier.go new file mode 100644 index 00000000000..c3a3094cec0 --- /dev/null +++ b/test/testutils/applier.go @@ -0,0 +1,67 @@ +package testutils + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" +) + +// Applies fixture YAMLs directly under the provided path in alpha-sorted order. +// Does not crawl the directory, instead it only looks at the files present at path. +func ApplyFixtures(path string, cfg *rest.Config) error { + files, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("reading path %s: %w", path, err) + } + + c, err := client.New(cfg, client.Options{}) + if err != nil { + return fmt.Errorf("creating client: %w", err) + } + + sorted := make([]string, 0, len(files)) + for _, entry := range files { + if entry.IsDir() { + continue + } + sorted = append(sorted, entry.Name()) + } + sort.StringSlice(sorted).Sort() + + for _, entry := range sorted { + b, err := os.ReadFile(filepath.Join(path, entry)) + if err != nil { + return fmt.Errorf("reading file %s: %w", entry, err) + } + + desired := unstructured.Unstructured{} + if err := yaml.Unmarshal(b, &desired); err != nil { + return fmt.Errorf("parsing file %s: %w", entry, err) + } + + u := unstructured.Unstructured{} + u.SetGroupVersionKind(desired.GroupVersionKind()) + u.SetName(desired.GetName()) + u.SetNamespace(desired.GetNamespace()) + _, err = controllerutil.CreateOrUpdate(context.Background(), c, &u, func() error { + resourceVersion := u.GetResourceVersion() + desired.DeepCopyInto(&u) + u.SetResourceVersion(resourceVersion) + + return nil + }) + if err != nil { + return fmt.Errorf("creating %v %s: %w", u.GroupVersionKind(), u.GetName(), err) + } + } + + return nil +} From 46dfde2e611c4d92d357876b697e4fc1bfa87d8e Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 3 Oct 2023 19:06:42 +0000 Subject: [PATCH 04/42] enable syncset crd Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- config/crd/kustomization.yaml | 2 +- .../syncset-customresourcedefinition.yaml | 48 ++++++++++++++++++ manifest_staging/deploy/gatekeeper.yaml | 49 +++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 9d6344716ec..d711fe22595 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,7 +3,7 @@ # It should be run by config/default resources: - bases/config.gatekeeper.sh_configs.yaml -#- bases/syncset.gatekeeper.sh_syncsets.yaml +- bases/syncset.gatekeeper.sh_syncsets.yaml - bases/status.gatekeeper.sh_constraintpodstatuses.yaml - bases/status.gatekeeper.sh_constrainttemplatepodstatuses.yaml - bases/status.gatekeeper.sh_mutatorpodstatuses.yaml diff --git a/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml b/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml new file mode 100644 index 00000000000..aac6d75125b --- /dev/null +++ b/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml @@ -0,0 +1,48 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + labels: + gatekeeper.sh/system: "yes" + name: syncsets.syncset.gatekeeper.sh +spec: + group: syncset.gatekeeper.sh + names: + kind: SyncSet + listKind: SyncSetList + plural: syncsets + singular: syncset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SyncSet is the Schema for the SyncSet API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + gvks: + items: + properties: + group: + type: string + kind: + type: string + version: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true diff --git a/manifest_staging/deploy/gatekeeper.yaml b/manifest_staging/deploy/gatekeeper.yaml index b68e60d2c5f..7c2ebde9194 100644 --- a/manifest_staging/deploy/gatekeeper.yaml +++ b/manifest_staging/deploy/gatekeeper.yaml @@ -3367,6 +3367,55 @@ spec: served: true storage: true --- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + labels: + gatekeeper.sh/system: "yes" + name: syncsets.syncset.gatekeeper.sh +spec: + group: syncset.gatekeeper.sh + names: + kind: SyncSet + listKind: SyncSetList + plural: syncsets + singular: syncset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SyncSet is the Schema for the SyncSet API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + gvks: + items: + properties: + group: + type: string + kind: + type: string + version: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true +--- apiVersion: v1 kind: ServiceAccount metadata: From 686af5ddc908cfe3ca0f3558175692d145f36bc8 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 5 Oct 2023 00:56:40 +0000 Subject: [PATCH 05/42] interpret registrar err Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 53 +++++- pkg/cachemanager/cachemanager_test.go | 230 ++++++++++++++++++++++---- pkg/watch/errorlist.go | 31 +++- pkg/watch/manager.go | 4 +- 4 files changed, 282 insertions(+), 36 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index bc2cc1c0919..6f3ea7c4756 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -2,10 +2,12 @@ package cachemanager import ( "context" + "errors" "fmt" "sync" "time" + "github.com/go-logr/logr" "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" @@ -141,7 +143,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // may become unreferenced and need to be deleted; this will be handled async // in the manageCache loop. - if err := c.replaceWatchSet(ctx); err != nil { + if err := c.replaceWatchSet(ctx, newGVKs); err != nil { return fmt.Errorf("error watching new gvks: %w", err) } @@ -150,7 +152,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // replaceWatchSet looks at the gvksToSync and makes changes to the registrar's watch set. // Assumes caller has lock. On error, actual watch state may not align with intended watch state. -func (c *CacheManager) replaceWatchSet(ctx context.Context) error { +func (c *CacheManager) replaceWatchSet(ctx context.Context, pertinentGVKs []schema.GroupVersionKind) error { newWatchSet := watch.NewSet() newWatchSet.Add(c.gvksToSync.GVKs()...) @@ -167,7 +169,50 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { innerError = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) }) - return innerError + if err := interpretErr(log, innerError, pertinentGVKs); err != nil { + return err + } + + return nil +} + +// interpretErr looks at the error e and unwraps it looking to find an error FailingGVKs() in the error chain. +// If found and there is an intersection with the given gvks parameter with the failing gvks, we return an error +// that is meant to bubble up to controllers. If there is no intersection, we log the error so as not to fully +// swallow any issues. If no error implemeting FailingGVKs() is found, we consider that to be a "global error" +// and it is returned. +func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) error { + if e == nil { + return nil + } + + var f interface { + FailingGVKs() []schema.GroupVersionKind + Error() string + } + + // search for the first error that implements the FailingGVKs() interface + if errors.As(e, &f) { + // this error includes failing gvks; let's match against the original list + failedGvks := watch.NewSet() + failedGvks.Add(f.FailingGVKs()...) + gvksSet := watch.NewSet() + gvksSet.Add(gvks...) + + common := failedGvks.Intersection(gvksSet) + if common.Size() > 0 { + // then this error is pertinent to the gvks and needs to be returned + return e + } + + // if no intersection, this error is not about the gvks in this request + // but we still log it for visibility + logger.Info("encountered unrelated error when replacing watch set", "error", e) + return nil + } + + // otherwise, this is a "global error" + return e } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. @@ -179,7 +224,7 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke return fmt.Errorf("internal error removing source: %w", err) } - if err := c.replaceWatchSet(ctx); err != nil { + if err := c.replaceWatchSet(ctx, []schema.GroupVersionKind{}); err != nil { return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 3e7fa9bbeec..c142e412c3a 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -2,8 +2,11 @@ package cachemanager import ( "context" + "errors" + "fmt" "testing" + "github.com/go-logr/logr" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" @@ -25,6 +28,17 @@ import ( var cfg *rest.Config +var ( + configMapGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + nsGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} + nonExistentGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "DoesNotExist"} + + sourceA = aggregator.Key{Source: "a", ID: "source"} + sourceB = aggregator.Key{Source: "b", ID: "source"} + sourceC = aggregator.Key{Source: "c", ID: "source"} +) + func TestMain(m *testing.M) { testutils.StartControlPlane(m, &cfg, 2) } @@ -72,7 +86,6 @@ func makeCacheManager(t *testing.T) (*CacheManager, context.Context) { } func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} dataClientForTest := func() CFDataClient { cfdc := &fakes.FakeCfClient{} @@ -354,14 +367,10 @@ func TestCacheManager_RemoveObject(t *testing.T) { // TestCacheManager_UpsertSource tests that we can modify the gvk aggregator and watched set when adding a new source. func TestCacheManager_UpsertSource(t *testing.T) { - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - sourceA := aggregator.Key{Source: "a", ID: "source"} - sourceB := aggregator.Key{Source: "b", ID: "source"} - type sourcesAndGvk struct { source aggregator.Key gvks []schema.GroupVersionKind + err error } tcs := []struct { @@ -422,7 +431,21 @@ func TestCacheManager_UpsertSource(t *testing.T) { expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, }, { - name: "add two sources with overlapping gvks", + name: "add two sources with fully overlapping gvks", + sourcesAndGvks: []sourcesAndGvk{ + { + source: sourceA, + gvks: []schema.GroupVersionKind{podGVK}, + }, + { + source: sourceB, + gvks: []schema.GroupVersionKind{podGVK}, + }, + }, + expectedGVKs: []schema.GroupVersionKind{podGVK}, + }, + { + name: "add two sources with partially overlapping gvks", sourcesAndGvks: []sourcesAndGvk{ { source: sourceA, @@ -435,6 +458,28 @@ func TestCacheManager_UpsertSource(t *testing.T) { }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, }, + { + name: "add two sources where one fails to establish all watches", + sourcesAndGvks: []sourcesAndGvk{ + { + source: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, + }, + { + source: sourceB, + gvks: []schema.GroupVersionKind{podGVK, nonExistentGVK}, + // UpsertSource will err out because of nonExistentGVK + err: errors.New("error for gvk: /v1, Kind=DoesNotExist: adding watch for /v1, Kind=DoesNotExist getting informer for kind: /v1, Kind=DoesNotExist no matches for kind \"DoesNotExist\" in version \"v1\""), + }, + { + source: sourceC, + gvks: []schema.GroupVersionKind{nsGVK}, + // without error interpretation, this upsert would fail because we added a + // non existent gvk previously. + }, + }, + expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK, nonExistentGVK, nsGVK}, + }, } for _, tc := range tcs { @@ -442,7 +487,11 @@ func TestCacheManager_UpsertSource(t *testing.T) { cacheManager, ctx := makeCacheManager(t) for _, sourceAndGVK := range tc.sourcesAndGvks { - require.NoError(t, cacheManager.UpsertSource(ctx, sourceAndGVK.source, sourceAndGVK.gvks)) + if sourceAndGVK.err != nil { + require.ErrorContains(t, cacheManager.UpsertSource(ctx, sourceAndGVK.source, sourceAndGVK.gvks), sourceAndGVK.err.Error(), fmt.Sprintf("while upserting source: %s", sourceAndGVK.source)) + } else { + require.NoError(t, cacheManager.UpsertSource(ctx, sourceAndGVK.source, sourceAndGVK.gvks), fmt.Sprintf("while upserting source: %s", sourceAndGVK.source)) + } } require.ElementsMatch(t, cacheManager.watchedSet.Items(), tc.expectedGVKs) @@ -453,11 +502,6 @@ func TestCacheManager_UpsertSource(t *testing.T) { // TestCacheManager_RemoveSource tests that we can modify the gvk aggregator when removing a source. func TestCacheManager_RemoveSource(t *testing.T) { - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - sourceA := aggregator.Key{Source: "a", ID: "source"} - sourceB := aggregator.Key{Source: "b", ID: "source"} - tcs := []struct { name string seed func(c *CacheManager) @@ -474,7 +518,7 @@ func TestCacheManager_RemoveSource(t *testing.T) { expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { - name: "remove overlapping source", + name: "remove fully overlapping source", seed: func(c *CacheManager) { require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK})) @@ -482,6 +526,15 @@ func TestCacheManager_RemoveSource(t *testing.T) { sourcesToRemove: []aggregator.Key{sourceB}, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, + { + name: "remove partially overlapping source", + seed: func(c *CacheManager) { + require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) + require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) + }, + sourcesToRemove: []aggregator.Key{sourceA}, + expectedGVKs: []schema.GroupVersionKind{podGVK, configMapGVK}, + }, { name: "remove non existing source", seed: func(c *CacheManager) { @@ -490,6 +543,35 @@ func TestCacheManager_RemoveSource(t *testing.T) { sourcesToRemove: []aggregator.Key{sourceB}, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, + { + name: "remove source w a non existing gvk", + seed: func(c *CacheManager) { + require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{nonExistentGVK})) + }, + sourcesToRemove: []aggregator.Key{sourceA}, + expectedGVKs: []schema.GroupVersionKind{}, + }, + { + name: "remove source from a watch set w a non existing gvk", + seed: func(c *CacheManager) { + require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{nonExistentGVK})) + require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK})) + }, + // without interpreting the error, removing a source that doesn't reference a non existent gvk + // would still error out. + sourcesToRemove: []aggregator.Key{sourceB}, + expectedGVKs: []schema.GroupVersionKind{nonExistentGVK}, + }, + { + name: "remove source w non existent gvk from a watch set w a remaining non existing gvk", + seed: func(c *CacheManager) { + require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{nonExistentGVK})) + require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{nonExistentGVK})) + }, + // without interpreting the error, removing a source here would error out. + sourcesToRemove: []aggregator.Key{sourceB}, + expectedGVKs: []schema.GroupVersionKind{nonExistentGVK}, + }, } for _, tc := range tcs { @@ -504,20 +586,6 @@ func TestCacheManager_RemoveSource(t *testing.T) { require.ElementsMatch(t, cm.gvksToSync.GVKs(), tc.expectedGVKs) }) } - cacheManager, ctx := makeCacheManager(t) - - // seed the gvk aggregator - require.NoError(t, cacheManager.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) - require.NoError(t, cacheManager.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) - - // removing a source that is not the only one referencing a gvk ... - require.NoError(t, cacheManager.RemoveSource(ctx, sourceB)) - // ... should not remove any gvks that are still referenced by other sources - require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) - require.False(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) - - require.NoError(t, cacheManager.RemoveSource(ctx, sourceA)) - require.False(t, cacheManager.gvksToSync.IsPresent(podGVK)) } func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Unstructured { @@ -537,3 +605,109 @@ func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Uns } return u } + +func Test_interpretErr(t *testing.T) { + logger := logr.Discard() + gvk1 := schema.GroupVersionKind{Group: "g1", Version: "v1", Kind: "k1"} + gvk2 := schema.GroupVersionKind{Group: "g2", Version: "v2", Kind: "k2"} + + cases := []struct { + name string + inputErr error + inputGVK []schema.GroupVersionKind + expected error + }{ + { + name: "nil err", + inputErr: nil, + expected: nil, + }, + { + name: "intersection exists, wrapped", + inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), + inputGVK: []schema.GroupVersionKind{gvk1}, + expected: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), + }, + { + name: "intersection exists, unwrapped", + inputErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + inputGVK: []schema.GroupVersionKind{gvk1}, + expected: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + }, + { + name: "intersection does not exist", + inputErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + inputGVK: []schema.GroupVersionKind{gvk2}, + expected: nil, + }, + { + name: "intersection does not exist, GVKs is empty", + inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), + inputGVK: []schema.GroupVersionKind{}, + expected: nil, + }, + { + name: "global error, gvks inputed", + inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), + inputGVK: []schema.GroupVersionKind{gvk1}, + expected: fmt.Errorf("some err: %w", errors.New("some other err")), + }, + { + name: "global error, no gvks inputed", + inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), + inputGVK: []schema.GroupVersionKind{}, + expected: fmt.Errorf("some err: %w", errors.New("some other err")), + }, + { + name: "global unwrapped error, gvks inputed", + inputErr: errors.New("some err"), + inputGVK: []schema.GroupVersionKind{gvk1}, + expected: errors.New("some err"), + }, + { + name: "nested gvk error, intersection", + inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + inputGVK: []schema.GroupVersionKind{gvk1}, + expected: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + }, + { + name: "nested gvk error, no intersection", + inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + inputGVK: []schema.GroupVersionKind{gvk2}, + expected: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := interpretErr(logger, tc.inputErr, tc.inputGVK) + + if tc.expected != nil { + require.Equal(t, tc.expected.Error(), got.Error()) + } else { + require.Nil(t, got, fmt.Sprintf("expected nil error, got: %s", got)) + } + }) + } +} + +type gvkError struct { + gvks []schema.GroupVersionKind + err error +} + +func (e *gvkError) Error() string { + return fmt.Sprintf("failing gvks: %v", e.gvks) +} + +func (e *gvkError) FailingGVKs() []schema.GroupVersionKind { + return e.gvks +} + +func (e *gvkError) Unwrap() error { + if e.err != nil { + return e.err + } + + return nil +} diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index 1c2f024801a..4a87a6a9293 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -15,10 +15,28 @@ limitations under the License. package watch -import "strings" +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/runtime/schema" +) // errorList is an error that aggregates multiple errors. -type errorList []error +type errorList []GVKError + +type GVKError struct { + err error + gvk schema.GroupVersionKind +} + +func (w GVKError) String() string { + return w.Error() +} + +func (w GVKError) Error() string { + return fmt.Sprintf("error for gvk: %s: %s", w.gvk, w.err.Error()) +} func (e errorList) String() string { return e.Error() @@ -34,3 +52,12 @@ func (e errorList) Error() string { } return builder.String() } + +func (e errorList) FailingGVKs() []schema.GroupVersionKind { + gvks := make([]schema.GroupVersionKind, len(e)) + for _, err := range e { + gvks = append(gvks, err.gvk) + } + + return gvks +} diff --git a/pkg/watch/manager.go b/pkg/watch/manager.go index 7858fa42d78..a87fdaf0809 100644 --- a/pkg/watch/manager.go +++ b/pkg/watch/manager.go @@ -263,7 +263,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { continue } if err := wm.doRemoveWatch(ctx, r, gvk); err != nil { - errlist = append(errlist, fmt.Errorf("removing watch for %+v %w", gvk, err)) + errlist = append(errlist, GVKError{gvk: gvk, err: fmt.Errorf("removing watch for %+v %w", gvk, err)}) } } @@ -273,7 +273,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { continue } if err := wm.doAddWatch(ctx, r, gvk); err != nil { - errlist = append(errlist, fmt.Errorf("adding watch for %+v %w", gvk, err)) + errlist = append(errlist, GVKError{gvk: gvk, err: fmt.Errorf("adding watch for %+v %w", gvk, err)}) } } From 7e94cddea630da1e67a4e73d3b443752406ecc8e Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 5 Oct 2023 23:22:50 +0000 Subject: [PATCH 06/42] review: move EM, rename to EP plus naming, style, etc Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- main.go | 6 +- pkg/cachemanager/cachemanager.go | 13 ----- pkg/controller/config/config_controller.go | 5 +- pkg/controller/syncset/syncset_controller.go | 56 +++---------------- pkg/readiness/object_tracker.go | 6 +- .../pruner/pruner.go} | 20 +++---- .../pruner/pruner_test.go} | 8 +-- .../00-namespace.yaml | 0 .../20-syncset-1.yaml | 0 .../syncsets-config-disjoint/21-config.yaml | 0 .../syncsets-overlapping/20-syncset-1.yaml | 0 .../syncsets-overlapping/20-syncset-2.yaml | 0 .../syncsets-overlapping/20-syncset-3.yaml | 0 .../testdata/syncsets-resources/00-gk-ns.yaml | 0 .../syncsets-resources/11-config.yaml | 0 .../syncsets-resources/15-configmap-1.yaml | 0 .../syncsets-resources/20-syncset-1.yaml | 0 .../syncsets-resources/20-syncset-2.yaml | 0 pkg/readiness/ready_tracker.go | 16 ++---- 19 files changed, 33 insertions(+), 97 deletions(-) rename pkg/{expectationsmgr/expectationsmgr.go => readiness/pruner/pruner.go} (60%) rename pkg/{expectationsmgr/expectationsmgr_test.go => readiness/pruner/pruner_test.go} (97%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-config-disjoint/00-namespace.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-config-disjoint/20-syncset-1.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-config-disjoint/21-config.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-overlapping/20-syncset-1.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-overlapping/20-syncset-2.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-overlapping/20-syncset-3.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-resources/00-gk-ns.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-resources/11-config.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-resources/15-configmap-1.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-resources/20-syncset-1.yaml (100%) rename pkg/{expectationsmgr => readiness/pruner}/testdata/syncsets-resources/20-syncset-2.yaml (100%) diff --git a/main.go b/main.go index 216afb6b5a6..6099b82aebc 100644 --- a/main.go +++ b/main.go @@ -47,13 +47,13 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/controller" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" - "github.com/open-policy-agent/gatekeeper/v3/pkg/expectationsmgr" "github.com/open-policy-agent/gatekeeper/v3/pkg/externaldata" "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/operations" "github.com/open-policy-agent/gatekeeper/v3/pkg/pubsub" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness/pruner" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/upgrade" @@ -477,8 +477,8 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle return err } - em := expectationsmgr.NewExpecationsManager(cm, tracker) - go em.Run(ctx) + p := pruner.NewExpecationsPruner(cm, tracker) + go p.Run(ctx) opts := controller.Dependencies{ CFClient: client, diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 6f3ea7c4756..dde264e2a45 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -61,8 +61,6 @@ type CacheManager struct { registrar *watch.Registrar backgroundManagementTicker time.Ticker reader client.Reader - - started bool } // CFDataClient is an interface for caching data. @@ -108,21 +106,10 @@ func NewCacheManager(config *Config) (*CacheManager, error) { func (c *CacheManager) Start(ctx context.Context) error { go c.manageCache(ctx) - c.mu.Lock() - c.started = true - c.mu.Unlock() - <-ctx.Done() return nil } -func (c *CacheManager) Started() bool { - c.mu.RLock() - defer c.mu.RUnlock() - - return c.started -} - // UpsertSource adjusts the watched set of gvks according to the newGVKs passed in // for a given sourceKey. Callers are responsible for retrying on error. func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index bf2c75e571f..98fe5d17890 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -177,8 +177,6 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque // sync anything gvksToSync := []schema.GroupVersionKind{} if exists && instance.GetDeletionTimestamp().IsZero() { - r.tracker.For(instance.GroupVersionKind()).Observe(instance) - for _, entry := range instance.Spec.Sync.SyncOnly { gvk := schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind} gvksToSync = append(gvksToSync, gvk) @@ -197,12 +195,15 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque r.tracker.DisableStats() } + r.tracker.For(instance.GroupVersionKind()).Observe(instance) + r.cacheManager.ExcludeProcesses(newExcluder) configSourceKey := aggregator.Key{Source: "config", ID: request.NamespacedName.String()} if err := r.cacheManager.UpsertSource(ctx, configSourceKey, gvksToSync); err != nil { return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) } + // r.tracker.For(instance.GroupVersionKind()).Observe(instance) return reconcile.Result{}, nil } diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index f3859445d1b..946dac21d80 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -75,8 +75,7 @@ func newReconciler(mgr manager.Manager, cm *cm.CacheManager, cs *watch.Controlle } return &ReconcileSyncSet{ - reader: mgr.GetCache(), - writer: mgr.GetClient(), + reader: mgr.GetClient(), scheme: mgr.GetScheme(), cs: cs, cacheManager: cm, @@ -103,7 +102,6 @@ var _ reconcile.Reconciler = &ReconcileSyncSet{} // ReconcileSyncSet reconciles a SyncSet object. type ReconcileSyncSet struct { reader client.Reader - writer client.Writer scheme *runtime.Scheme cacheManager *cm.CacheManager @@ -123,8 +121,8 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ syncsetTr := r.tracker.For(syncsetGVK) exists := true - instance := &syncsetv1alpha1.SyncSet{} - err := r.reader.Get(ctx, request.NamespacedName, instance) + syncset := &syncsetv1alpha1.SyncSet{} + err := r.reader.Get(ctx, request.NamespacedName, syncset) if err != nil { if errors.IsNotFound(err) { exists = false @@ -134,24 +132,11 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ } } - if exists { - // Actively remove a finalizer. This should automatically remove - // the finalizer over time even if state teardown didn't work correctly - // after a deprecation period, all finalizer code can be removed. - if hasFinalizer(instance) { - removeFinalizer(instance) - if err := r.writer.Update(ctx, instance); err != nil { - return reconcile.Result{}, err - } - } - } - gvks := make([]schema.GroupVersionKind, 0) - if exists && instance.GetDeletionTimestamp().IsZero() { - log.Info("handling SyncSet update", "instance", instance) - syncsetTr.Observe(instance) + if exists && syncset.GetDeletionTimestamp().IsZero() { + log.V(logging.DebugLevel).Info("handling SyncSet update", "instance", syncset) - for _, entry := range instance.Spec.GVKs { + for _, entry := range syncset.Spec.GVKs { gvks = append(gvks, schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind}) } } @@ -161,32 +146,7 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{Requeue: true}, fmt.Errorf("synceset-controller: error changing watches: %w", err) } - return reconcile.Result{}, nil -} - -func containsString(s string, items []string) bool { - for _, item := range items { - if item == s { - return true - } - } - return false -} - -func removeString(s string, items []string) []string { - var rval []string - for _, item := range items { - if item != s { - rval = append(rval, item) - } - } - return rval -} - -func hasFinalizer(instance *syncsetv1alpha1.SyncSet) bool { - return containsString(finalizerName, instance.GetFinalizers()) -} + syncsetTr.Observe(syncset) -func removeFinalizer(instance *syncsetv1alpha1.SyncSet) { - instance.SetFinalizers(removeString(finalizerName, instance.GetFinalizers())) + return reconcile.Result{}, nil } diff --git a/pkg/readiness/object_tracker.go b/pkg/readiness/object_tracker.go index ee09f5f4ff2..d7aa0929837 100644 --- a/pkg/readiness/object_tracker.go +++ b/pkg/readiness/object_tracker.go @@ -243,16 +243,12 @@ func (t *objectTracker) Observe(o runtime.Object) { _, wasExpecting := t.expect[k] switch { case wasExpecting: - log.V(1).Info("[readiness] observing expectation", "gvk", o.GetObjectKind().GroupVersionKind()) - // Satisfy existing expectation delete(t.seen, k) delete(t.expect, k) t.satisfied[k] = struct{}{} return case !wasExpecting && t.populated: - log.V(1).Info("[readiness] not expecting anymore", "gvk", o.GetObjectKind().GroupVersionKind()) - // Not expecting and no longer accepting expectations. // No need to track. delete(t.seen, k) @@ -305,7 +301,7 @@ func (t *objectTracker) Satisfied() bool { if !needMutate { // Read lock to prevent concurrent read/write while logging readiness state. t.mu.RLock() - log.V(1).Info("readiness state unsatisfied", "gvk", t.gvk, "satisfied", fmt.Sprintf("%d/%d", len(t.satisfied), len(t.expect)+len(t.satisfied))) + log.V(1).Info("readiness state", "gvk", t.gvk, "satisfied", fmt.Sprintf("%d/%d", len(t.satisfied), len(t.expect)+len(t.satisfied)), "populated", t.populated) t.mu.RUnlock() return false } diff --git a/pkg/expectationsmgr/expectationsmgr.go b/pkg/readiness/pruner/pruner.go similarity index 60% rename from pkg/expectationsmgr/expectationsmgr.go rename to pkg/readiness/pruner/pruner.go index 3bbe22c1b21..8dd0ac25982 100644 --- a/pkg/expectationsmgr/expectationsmgr.go +++ b/pkg/readiness/pruner/pruner.go @@ -1,4 +1,4 @@ -package expectationsmgr +package pruner import ( "context" @@ -11,22 +11,22 @@ import ( const tickDuration = 3 * time.Second -// ExpectationsMgr fires after data expectations have been populated in the ready Tracker and runs -// until the Tracker is satisfeid or the process is exiting. It removes Data expectations for any +// ExpectationsPruner fires after sync expectations have been satisfied in the ready Tracker and runs +// until the overall Tracker is satisfied. It removes Data expectations for any // GVKs that are expected in the Tracker but not watched by the CacheManager. -type ExpectationsMgr struct { +type ExpectationsPruner struct { cacheMgr *cachemanager.CacheManager tracker *readiness.Tracker } -func NewExpecationsManager(cm *cachemanager.CacheManager, rt *readiness.Tracker) *ExpectationsMgr { - return &ExpectationsMgr{ +func NewExpecationsPruner(cm *cachemanager.CacheManager, rt *readiness.Tracker) *ExpectationsPruner { + return &ExpectationsPruner{ cacheMgr: cm, tracker: rt, } } -func (e *ExpectationsMgr) Run(ctx context.Context) { +func (e *ExpectationsPruner) Run(ctx context.Context) { ticker := time.NewTicker(tickDuration) for { select { @@ -38,10 +38,8 @@ func (e *ExpectationsMgr) Run(ctx context.Context) { // further manage the data sync expectations. return } - if !(e.tracker.DataPopulated() && e.cacheMgr.Started()) { - // we have to wait on data expectations to be populated - // and for the cachemanager to have been started by the - // controller manager. + if !e.tracker.SyncSourcesSatisfied() { + // not yet ready to prune data expectations. break } diff --git a/pkg/expectationsmgr/expectationsmgr_test.go b/pkg/readiness/pruner/pruner_test.go similarity index 97% rename from pkg/expectationsmgr/expectationsmgr_test.go rename to pkg/readiness/pruner/pruner_test.go index b2c737cdc9b..116c59b89da 100644 --- a/pkg/expectationsmgr/expectationsmgr_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -1,4 +1,4 @@ -package expectationsmgr +package pruner import ( "context" @@ -46,10 +46,10 @@ var ( ) func TestMain(m *testing.M) { - testutils.StartControlPlane(m, &cfg, 2) + testutils.StartControlPlane(m, &cfg, 3) } -func setupTest(ctx context.Context, t *testing.T, startControllers bool) (*ExpectationsMgr, client.Client) { +func setupTest(ctx context.Context, t *testing.T, startControllers bool) (*ExpectationsPruner, client.Client) { t.Helper() mgr, wm := testutils.SetupManager(t, cfg) @@ -105,7 +105,7 @@ func setupTest(ctx context.Context, t *testing.T, startControllers bool) (*Expec testutils.StartManager(ctx, t, mgr) - return &ExpectationsMgr{ + return &ExpectationsPruner{ cacheMgr: cm, tracker: tracker, }, c diff --git a/pkg/expectationsmgr/testdata/syncsets-config-disjoint/00-namespace.yaml b/pkg/readiness/pruner/testdata/syncsets-config-disjoint/00-namespace.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-config-disjoint/00-namespace.yaml rename to pkg/readiness/pruner/testdata/syncsets-config-disjoint/00-namespace.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-config-disjoint/20-syncset-1.yaml b/pkg/readiness/pruner/testdata/syncsets-config-disjoint/20-syncset-1.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-config-disjoint/20-syncset-1.yaml rename to pkg/readiness/pruner/testdata/syncsets-config-disjoint/20-syncset-1.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-config-disjoint/21-config.yaml b/pkg/readiness/pruner/testdata/syncsets-config-disjoint/21-config.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-config-disjoint/21-config.yaml rename to pkg/readiness/pruner/testdata/syncsets-config-disjoint/21-config.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-1.yaml b/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-1.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-1.yaml rename to pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-1.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-2.yaml b/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-2.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-2.yaml rename to pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-2.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-3.yaml b/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-3.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-overlapping/20-syncset-3.yaml rename to pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-3.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/00-gk-ns.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/00-gk-ns.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-resources/00-gk-ns.yaml rename to pkg/readiness/pruner/testdata/syncsets-resources/00-gk-ns.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/11-config.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/11-config.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-resources/11-config.yaml rename to pkg/readiness/pruner/testdata/syncsets-resources/11-config.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/15-configmap-1.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/15-configmap-1.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-resources/15-configmap-1.yaml rename to pkg/readiness/pruner/testdata/syncsets-resources/15-configmap-1.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-1.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-1.yaml rename to pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml diff --git a/pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-2.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-2.yaml similarity index 100% rename from pkg/expectationsmgr/testdata/syncsets-resources/20-syncset-2.yaml rename to pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-2.yaml diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index a5adcae8bf6..d2abc275dc6 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -185,17 +185,15 @@ func (t *Tracker) For(gvk schema.GroupVersionKind) Expectations { func (t *Tracker) ForData(gvk schema.GroupVersionKind) Expectations { // Avoid new data trackers after data expectations have been fully populated. // Race is ok here - extra trackers will only consume some unneeded memory. - if t.DataPopulated() && !t.data.Has(gvk) { + if t.config.Populated() && t.syncsets.Populated() && !t.data.Has(gvk) { // Return throw-away tracker instead. return noopExpectations{} } return t.data.Get(gvk) } +// Returns the GVKs for which the Tracker has data expectations. func (t *Tracker) DataGVKs() []schema.GroupVersionKind { - t.mu.RLock() - defer t.mu.RUnlock() - return t.data.Keys() } @@ -400,13 +398,9 @@ func (t *Tracker) Populated() bool { return validationPopulated && t.config.Populated() && mutationPopulated && externalDataProviderPopulated && t.syncsets.Populated() } -func (t *Tracker) DataPopulated() bool { - dataPopulated := true - if operations.HasValidationOperations() { - dataPopulated = t.data.Populated() - } - - return dataPopulated && t.config.Populated() && t.syncsets.Populated() +// Returns whether both the Config and all SyncSet expectations have been Satisfied. +func (t *Tracker) SyncSourcesSatisfied() bool { + return t.config.Satisfied() && t.syncsets.Satisfied() } // collectForObjectTracker identifies objects that are unsatisfied for the provided From 31979ace98ce37ded16d81d2a722ab7b2f316f88 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 6 Oct 2023 01:06:13 +0000 Subject: [PATCH 07/42] review: remove waitgroup Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/readiness/ready_tracker.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index d2abc275dc6..6d5deb57a6a 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -724,15 +724,12 @@ func (t *Tracker) trackConstraintTemplates(ctx context.Context) error { // and any SyncSet resources present on the cluster. // Works best effort and fails-open if the a resource cannot be fetched or does not exist. func (t *Tracker) trackSyncSources(ctx context.Context) error { - var wg sync.WaitGroup defer func() { t.config.ExpectationsDone() log.V(1).Info("config expectations populated") t.syncsets.ExpectationsDone() log.V(1).Info("syncset expectations populated") - - wg.Wait() }() handled := make(map[schema.GroupVersionKind]struct{}) @@ -791,9 +788,7 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { // Set expectations for individual cached resources dt := t.ForData(g) - wg.Add(1) go func() { - defer wg.Done() err := t.trackData(ctx, g, dt) if err != nil { log.Error(err, "aborted trackData", "gvk", g) From b29086771f1c270eae2113862131af5cb77913af Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Sat, 7 Oct 2023 00:11:25 +0000 Subject: [PATCH 08/42] smoll fix for test Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/readiness/ready_tracker_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/readiness/ready_tracker_test.go b/pkg/readiness/ready_tracker_test.go index 440f063eccb..c9ccc724866 100644 --- a/pkg/readiness/ready_tracker_test.go +++ b/pkg/readiness/ready_tracker_test.go @@ -584,8 +584,8 @@ func Test_Tracker_EdgeCases(t *testing.T) { for _, tt := range tts { t.Run(tt.name, func(t *testing.T) { testutils.Setenv(t, "POD_NAME", "no-pod") - require.NoError(t, applyFixtures("testdata"), "base fixtures config") - require.NoError(t, applyFixtures("testdata/config/bad-gvk"), fmt.Sprintf("test fixtures: %s", tt.fixturesPath)) + require.NoError(t, applyFixtures("testdata"), "base fixtures") + require.NoError(t, applyFixtures(tt.fixturesPath), fmt.Sprintf("test fixtures: %s", tt.fixturesPath)) mgr, wm := setupManager(t) cfClient := testutils.SetupDataClient(t) From 2192d6501e822f59d7441d4faeff083184b293dd Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 9 Oct 2023 23:05:58 +0000 Subject: [PATCH 09/42] use logging constant Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/readiness/ready_tracker.go | 95 +++++++++++++++++----------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index 6d5deb57a6a..e8e0f93f035 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -31,6 +31,7 @@ import ( mutationsv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/mutations/v1alpha1" syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" + "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" "github.com/open-policy-agent/gatekeeper/v3/pkg/operations" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/pkg/errors" @@ -206,7 +207,7 @@ func (t *Tracker) templateCleanup(ct *templates.ConstraintTemplate) { // CancelTemplate stops expecting the provided ConstraintTemplate and associated Constraints. func (t *Tracker) CancelTemplate(ct *templates.ConstraintTemplate) { - log.V(1).Info("cancel tracking for template", "namespace", ct.GetNamespace(), "name", ct.GetName()) + log.V(logging.DebugLevel).Info("cancel tracking for template", "namespace", ct.GetNamespace(), "name", ct.GetName()) t.templates.CancelExpect(ct) t.templateCleanup(ct) } @@ -215,7 +216,7 @@ func (t *Tracker) CancelTemplate(ct *templates.ConstraintTemplate) { // cancel the expectation for that CT and its associated Constraints if // no retries remain. func (t *Tracker) TryCancelTemplate(ct *templates.ConstraintTemplate) { - log.V(1).Info("try to cancel tracking for template", "namespace", ct.GetNamespace(), "name", ct.GetName()) + log.V(logging.DebugLevel).Info("try to cancel tracking for template", "namespace", ct.GetNamespace(), "name", ct.GetName()) if t.templates.TryCancelExpect(ct) { t.templateCleanup(ct) } @@ -223,7 +224,7 @@ func (t *Tracker) TryCancelTemplate(ct *templates.ConstraintTemplate) { // CancelData stops expecting data for the specified resource kind. func (t *Tracker) CancelData(gvk schema.GroupVersionKind) { - log.V(1).Info("cancel tracking for data", "gvk", gvk) + log.V(logging.DebugLevel).Info("cancel tracking for data", "gvk", gvk) t.data.Remove(gvk) } @@ -241,24 +242,24 @@ func (t *Tracker) Satisfied() bool { if !t.assignMetadata.Satisfied() || !t.assign.Satisfied() || !t.modifySet.Satisfied() || !t.assignImage.Satisfied() { return false } - log.V(1).Info("all expectations satisfied", "tracker", "assignMetadata") - log.V(1).Info("all expectations satisfied", "tracker", "assign") - log.V(1).Info("all expectations satisfied", "tracker", "modifySet") - log.V(1).Info("all expectations satisfied", "tracker", "assignImage") + log.V(logging.DebugLevel).Info("all expectations satisfied", "tracker", "assignMetadata") + log.V(logging.DebugLevel).Info("all expectations satisfied", "tracker", "assign") + log.V(logging.DebugLevel).Info("all expectations satisfied", "tracker", "modifySet") + log.V(logging.DebugLevel).Info("all expectations satisfied", "tracker", "assignImage") } if t.externalDataEnabled { if !t.externalDataProvider.Satisfied() { return false } - log.V(1).Info("all expectations satisfied", "tracker", "provider") + log.V(logging.DebugLevel).Info("all expectations satisfied", "tracker", "provider") } if t.expansionEnabled { if !t.expansions.Satisfied() { return false } - log.V(1).Info("all expectations satisfied", "tracker", "expansiontemplates") + log.V(logging.DebugLevel).Info("all expectations satisfied", "tracker", "expansiontemplates") } if operations.HasValidationOperations() { @@ -272,27 +273,27 @@ func (t *Tracker) Satisfied() bool { } } } - log.V(1).Info("all expectations satisfied", "tracker", "constraints") + log.V(logging.DebugLevel).Info("all expectations satisfied", "tracker", "constraints") if !t.config.Satisfied() { - log.V(1).Info("expectations unsatisfied", "tracker", "config") + log.V(logging.DebugLevel).Info("expectations unsatisfied", "tracker", "config") return false } if !t.syncsets.Satisfied() { - log.V(1).Info("expectations unsatisfied", "tracker", "syncset") + log.V(logging.DebugLevel).Info("expectations unsatisfied", "tracker", "syncset") return false } if operations.HasValidationOperations() { for _, gvk := range t.data.Keys() { if !t.data.Get(gvk).Satisfied() { - log.V(1).Info("expectations unsatisfied", "tracker", "data", "gvk", gvk) + log.V(logging.DebugLevel).Info("expectations unsatisfied", "tracker", "data", "gvk", gvk) return false } } } - log.V(1).Info("all expectations satisfied", "tracker", "data") + log.V(logging.DebugLevel).Info("all expectations satisfied", "tracker", "data") t.mu.Lock() defer t.mu.Unlock() @@ -411,12 +412,12 @@ func (t *Tracker) collectForObjectTracker(ctx context.Context, es Expectations, } if !es.Populated() { - log.V(1).Info("Expectations unpopulated, skipping collection", "tracker", trackerName) + log.V(logging.DebugLevel).Info("Expectations unpopulated, skipping collection", "tracker", trackerName) return nil } if es.Satisfied() { - log.V(1).Info("Expectations already satisfied, skipping collection", "tracker", trackerName) + log.V(logging.DebugLevel).Info("Expectations already satisfied, skipping collection", "tracker", trackerName) return nil } @@ -460,7 +461,7 @@ func (t *Tracker) collectForObjectTracker(ctx context.Context, es Expectations, u.SetNamespace(k.namespacedName.Namespace) u.SetGroupVersionKind(k.gvk) - log.V(1).Info("canceling expectations", "name", u.GetName(), "namespace", u.GetNamespace(), "gvk", u.GroupVersionKind(), "tracker", trackerName) + log.V(logging.DebugLevel).Info("canceling expectations", "name", u.GetName(), "namespace", u.GetNamespace(), "gvk", u.GroupVersionKind(), "tracker", trackerName) ot.CancelExpect(u) if cleanup != nil { cleanup(gvk) @@ -524,7 +525,7 @@ func (t *Tracker) collectInvalidExpectations(ctx context.Context) { func (t *Tracker) trackAssignMetadata(ctx context.Context) error { defer func() { t.assignMetadata.ExpectationsDone() - log.V(1).Info("AssignMetadata expectations populated") + log.V(logging.DebugLevel).Info("AssignMetadata expectations populated") _ = t.constraintTrackers.Wait() }() @@ -538,10 +539,10 @@ func (t *Tracker) trackAssignMetadata(ctx context.Context) error { if err := lister.List(ctx, assignMetadataList); err != nil { return fmt.Errorf("listing AssignMetadata: %w", err) } - log.V(1).Info("setting expectations for AssignMetadata", "AssignMetadata Count", len(assignMetadataList.Items)) + log.V(logging.DebugLevel).Info("setting expectations for AssignMetadata", "AssignMetadata Count", len(assignMetadataList.Items)) for index := range assignMetadataList.Items { - log.V(1).Info("expecting AssignMetadata", "name", assignMetadataList.Items[index].GetName()) + log.V(logging.DebugLevel).Info("expecting AssignMetadata", "name", assignMetadataList.Items[index].GetName()) t.assignMetadata.Expect(&assignMetadataList.Items[index]) } return nil @@ -550,7 +551,7 @@ func (t *Tracker) trackAssignMetadata(ctx context.Context) error { func (t *Tracker) trackAssign(ctx context.Context) error { defer func() { t.assign.ExpectationsDone() - log.V(1).Info("Assign expectations populated") + log.V(logging.DebugLevel).Info("Assign expectations populated") _ = t.constraintTrackers.Wait() }() @@ -563,10 +564,10 @@ func (t *Tracker) trackAssign(ctx context.Context) error { if err := lister.List(ctx, assignList); err != nil { return fmt.Errorf("listing Assign: %w", err) } - log.V(1).Info("setting expectations for Assign", "Assign Count", len(assignList.Items)) + log.V(logging.DebugLevel).Info("setting expectations for Assign", "Assign Count", len(assignList.Items)) for index := range assignList.Items { - log.V(1).Info("expecting Assign", "name", assignList.Items[index].GetName()) + log.V(logging.DebugLevel).Info("expecting Assign", "name", assignList.Items[index].GetName()) t.assign.Expect(&assignList.Items[index]) } return nil @@ -575,7 +576,7 @@ func (t *Tracker) trackAssign(ctx context.Context) error { func (t *Tracker) trackModifySet(ctx context.Context) error { defer func() { t.modifySet.ExpectationsDone() - log.V(1).Info("ModifySet expectations populated") + log.V(logging.DebugLevel).Info("ModifySet expectations populated") _ = t.constraintTrackers.Wait() }() @@ -588,10 +589,10 @@ func (t *Tracker) trackModifySet(ctx context.Context) error { if err := lister.List(ctx, modifySetList); err != nil { return fmt.Errorf("listing ModifySet: %w", err) } - log.V(1).Info("setting expectations for ModifySet", "ModifySet Count", len(modifySetList.Items)) + log.V(logging.DebugLevel).Info("setting expectations for ModifySet", "ModifySet Count", len(modifySetList.Items)) for index := range modifySetList.Items { - log.V(1).Info("expecting ModifySet", "name", modifySetList.Items[index].GetName()) + log.V(logging.DebugLevel).Info("expecting ModifySet", "name", modifySetList.Items[index].GetName()) t.modifySet.Expect(&modifySetList.Items[index]) } return nil @@ -600,7 +601,7 @@ func (t *Tracker) trackModifySet(ctx context.Context) error { func (t *Tracker) trackAssignImage(ctx context.Context) error { defer func() { t.assignImage.ExpectationsDone() - log.V(1).Info("AssignImage expectations populated") + log.V(logging.DebugLevel).Info("AssignImage expectations populated") _ = t.constraintTrackers.Wait() }() @@ -613,10 +614,10 @@ func (t *Tracker) trackAssignImage(ctx context.Context) error { if err := lister.List(ctx, assignImageList); err != nil { return fmt.Errorf("listing AssignImage: %w", err) } - log.V(1).Info("setting expectations for AssignImage", "AssignImage Count", len(assignImageList.Items)) + log.V(logging.DebugLevel).Info("setting expectations for AssignImage", "AssignImage Count", len(assignImageList.Items)) for index := range assignImageList.Items { - log.V(1).Info("expecting AssignImage", "name", assignImageList.Items[index].GetName()) + log.V(logging.DebugLevel).Info("expecting AssignImage", "name", assignImageList.Items[index].GetName()) t.assignImage.Expect(&assignImageList.Items[index]) } return nil @@ -625,7 +626,7 @@ func (t *Tracker) trackAssignImage(ctx context.Context) error { func (t *Tracker) trackExpansionTemplates(ctx context.Context) error { defer func() { t.expansions.ExpectationsDone() - log.V(1).Info("ExpansionTemplate expectations populated") + log.V(logging.DebugLevel).Info("ExpansionTemplate expectations populated") _ = t.constraintTrackers.Wait() }() @@ -638,10 +639,10 @@ func (t *Tracker) trackExpansionTemplates(ctx context.Context) error { if err := lister.List(ctx, expansionList); err != nil { return fmt.Errorf("listing ExpansionTemplate: %w", err) } - log.V(1).Info("setting expectations for ExpansionTemplate", "ExpansionTemplate Count", len(expansionList.Items)) + log.V(logging.DebugLevel).Info("setting expectations for ExpansionTemplate", "ExpansionTemplate Count", len(expansionList.Items)) for index := range expansionList.Items { - log.V(1).Info("expecting ExpansionTemplate", "name", expansionList.Items[index].GetName()) + log.V(logging.DebugLevel).Info("expecting ExpansionTemplate", "name", expansionList.Items[index].GetName()) t.expansions.Expect(&expansionList.Items[index]) } return nil @@ -650,7 +651,7 @@ func (t *Tracker) trackExpansionTemplates(ctx context.Context) error { func (t *Tracker) trackExternalDataProvider(ctx context.Context) error { defer func() { t.externalDataProvider.ExpectationsDone() - log.V(1).Info("Provider expectations populated") + log.V(logging.DebugLevel).Info("Provider expectations populated") _ = t.constraintTrackers.Wait() }() @@ -663,10 +664,10 @@ func (t *Tracker) trackExternalDataProvider(ctx context.Context) error { if err := lister.List(ctx, providerList); err != nil { return fmt.Errorf("listing Provider: %w", err) } - log.V(1).Info("setting expectations for Provider", "Provider Count", len(providerList.Items)) + log.V(logging.DebugLevel).Info("setting expectations for Provider", "Provider Count", len(providerList.Items)) for index := range providerList.Items { - log.V(1).Info("expecting Provider", "name", providerList.Items[index].GetName()) + log.V(logging.DebugLevel).Info("expecting Provider", "name", providerList.Items[index].GetName()) t.externalDataProvider.Expect(&providerList.Items[index]) } return nil @@ -675,7 +676,7 @@ func (t *Tracker) trackExternalDataProvider(ctx context.Context) error { func (t *Tracker) trackConstraintTemplates(ctx context.Context) error { defer func() { t.templates.ExpectationsDone() - log.V(1).Info("template expectations populated") + log.V(logging.DebugLevel).Info("template expectations populated") _ = t.constraintTrackers.Wait() }() @@ -686,7 +687,7 @@ func (t *Tracker) trackConstraintTemplates(ctx context.Context) error { return fmt.Errorf("listing templates: %w", err) } - log.V(1).Info("setting expectations for templates", "templateCount", len(templates.Items)) + log.V(logging.DebugLevel).Info("setting expectations for templates", "templateCount", len(templates.Items)) handled := make(map[schema.GroupVersionKind]bool, len(templates.Items)) for i := range templates.Items { @@ -694,7 +695,7 @@ func (t *Tracker) trackConstraintTemplates(ctx context.Context) error { // list is used for nothing else, so there is no danger of the object we // pass to templates.Expect() changing from underneath us. ct := &templates.Items[i] - log.V(1).Info("expecting template", "name", ct.GetName()) + log.V(logging.DebugLevel).Info("expecting template", "name", ct.GetName()) t.templates.Expect(ct) gvk := schema.GroupVersionKind{ @@ -726,10 +727,10 @@ func (t *Tracker) trackConstraintTemplates(ctx context.Context) error { func (t *Tracker) trackSyncSources(ctx context.Context) error { defer func() { t.config.ExpectationsDone() - log.V(1).Info("config expectations populated") + log.V(logging.DebugLevel).Info("config expectations populated") t.syncsets.ExpectationsDone() - log.V(1).Info("syncset expectations populated") + log.V(logging.DebugLevel).Info("syncset expectations populated") }() handled := make(map[schema.GroupVersionKind]struct{}) @@ -745,7 +746,7 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { log.Info("config resource is being deleted - skipping for readiness") } else { t.config.Expect(cfg) - log.V(1).Info("setting expectations for config", "configCount", 1) + log.V(logging.DebugLevel).Info("setting expectations for config", "configCount", 1) for _, entry := range cfg.Spec.Sync.SyncOnly { handled[schema.GroupVersionKind{ @@ -762,13 +763,13 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { if err := lister.List(ctx, syncsets); err != nil { log.Error(err, "listing syncsets") } else { - log.V(1).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) + log.V(logging.DebugLevel).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) for i := range syncsets.Items { syncset := syncsets.Items[i] t.syncsets.Expect(&syncset) - log.V(1).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) + log.V(logging.DebugLevel).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) for i := range syncset.Spec.GVKs { gvk := syncset.Spec.GVKs[i].ToGroupVersionKind() @@ -827,7 +828,7 @@ func (t *Tracker) getConfigResource(ctx context.Context) (*configv1alpha1.Config func (t *Tracker) trackData(ctx context.Context, gvk schema.GroupVersionKind, dt Expectations) error { defer func() { dt.ExpectationsDone() - log.V(1).Info("data expectations populated", "gvk", gvk) + log.V(logging.DebugLevel).Info("data expectations populated", "gvk", gvk) }() // List individual resources and expect observations of each in the sync controller. @@ -848,7 +849,7 @@ func (t *Tracker) trackData(ctx context.Context, gvk schema.GroupVersionKind, dt for i := range u.Items { item := &u.Items[i] dt.Expect(item) - log.V(1).Info("expecting data", "gvk", item.GroupVersionKind(), "namespace", item.GetNamespace(), "name", item.GetName()) + log.V(logging.DebugLevel).Info("expecting data", "gvk", item.GroupVersionKind(), "namespace", item.GetNamespace(), "name", item.GetName()) } return nil } @@ -858,7 +859,7 @@ func (t *Tracker) trackData(ctx context.Context, gvk schema.GroupVersionKind, dt func (t *Tracker) trackConstraints(ctx context.Context, gvk schema.GroupVersionKind, constraints Expectations) error { defer func() { constraints.ExpectationsDone() - log.V(1).Info("constraint expectations populated", "gvk", gvk) + log.V(logging.DebugLevel).Info("constraint expectations populated", "gvk", gvk) }() u := unstructured.UnstructuredList{} @@ -871,7 +872,7 @@ func (t *Tracker) trackConstraints(ctx context.Context, gvk schema.GroupVersionK for i := range u.Items { o := u.Items[i] constraints.Expect(&o) - log.V(1).Info("expecting Constraint", "gvk", gvk, "name", objectName(&o)) + log.V(logging.DebugLevel).Info("expecting Constraint", "gvk", gvk, "name", objectName(&o)) } return nil From 592e243d39fce391a5da606a0b1d9d3d42c7ed04 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 9 Oct 2023 23:15:11 +0000 Subject: [PATCH 10/42] add TryCancelData - rework interpretErr to return failing gvks - implement TryCancel for trackerMap Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 41 +++--- pkg/cachemanager/cachemanager_test.go | 105 +++++++------ pkg/controller/config/config_controller.go | 6 +- pkg/controller/syncset/syncset_controller.go | 2 + pkg/readiness/ready_tracker.go | 5 + pkg/readiness/ready_tracker_unit_test.go | 147 ++++++++++++++++--- pkg/readiness/tracker_map.go | 48 +++++- 7 files changed, 259 insertions(+), 95 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index dde264e2a45..dd95ad26d07 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -130,8 +130,12 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // may become unreferenced and need to be deleted; this will be handled async // in the manageCache loop. - if err := c.replaceWatchSet(ctx, newGVKs); err != nil { - return fmt.Errorf("error watching new gvks: %w", err) + err := c.replaceWatchSet(ctx) + if failedGVKs, interpreted := interpretErr(log, err, newGVKs); interpreted != nil { + for _, g := range failedGVKs { + c.tracker.TryCancelData(g) + } + return fmt.Errorf("error establishing watches: %w", interpreted) } return nil @@ -139,7 +143,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // replaceWatchSet looks at the gvksToSync and makes changes to the registrar's watch set. // Assumes caller has lock. On error, actual watch state may not align with intended watch state. -func (c *CacheManager) replaceWatchSet(ctx context.Context, pertinentGVKs []schema.GroupVersionKind) error { +func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet := watch.NewSet() newWatchSet.Add(c.gvksToSync.GVKs()...) @@ -156,21 +160,17 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context, pertinentGVKs []sche innerError = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) }) - if err := interpretErr(log, innerError, pertinentGVKs); err != nil { - return err - } - - return nil + return innerError } // interpretErr looks at the error e and unwraps it looking to find an error FailingGVKs() in the error chain. -// If found and there is an intersection with the given gvks parameter with the failing gvks, we return an error -// that is meant to bubble up to controllers. If there is no intersection, we log the error so as not to fully -// swallow any issues. If no error implemeting FailingGVKs() is found, we consider that to be a "global error" -// and it is returned. -func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) error { +// If found and there is an intersection with the given gvks parameter with the failing gvks, we return the failing +// gvks and an error that is meant to bubble up to controllers. If there is no intersection, we log the error +// so as not to fully swallow any issues. If no error implemeting FailingGVKs() is found, we consider that +// to be a "global error", so we return all gvks passed in and the error. +func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ([]schema.GroupVersionKind, error) { if e == nil { - return nil + return nil, nil } var f interface { @@ -189,17 +189,17 @@ func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) e common := failedGvks.Intersection(gvksSet) if common.Size() > 0 { // then this error is pertinent to the gvks and needs to be returned - return e + return common.Items(), e } // if no intersection, this error is not about the gvks in this request // but we still log it for visibility logger.Info("encountered unrelated error when replacing watch set", "error", e) - return nil + return nil, nil } // otherwise, this is a "global error" - return e + return gvks, e } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. @@ -211,8 +211,11 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke return fmt.Errorf("internal error removing source: %w", err) } - if err := c.replaceWatchSet(ctx, []schema.GroupVersionKind{}); err != nil { - return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) + err := c.replaceWatchSet(ctx) + if _, interpreted := interpretErr(log, err, []schema.GroupVersionKind{}); interpreted != nil { + // unlike UpsertSource, we cannot TryCancel or Cancel any expectations as these + // GVKs may be required by other sync sources. + return fmt.Errorf("error removing watches for source %v: %w", sourceKey, interpreted) } return nil diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index c142e412c3a..0123c1bcc95 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -612,80 +612,91 @@ func Test_interpretErr(t *testing.T) { gvk2 := schema.GroupVersionKind{Group: "g2", Version: "v2", Kind: "k2"} cases := []struct { - name string - inputErr error - inputGVK []schema.GroupVersionKind - expected error + name string + inputErr error + inputGVK []schema.GroupVersionKind + expectedErr error + expectedFailingGVKs []schema.GroupVersionKind }{ { - name: "nil err", - inputErr: nil, - expected: nil, + name: "nil err", + inputErr: nil, + expectedErr: nil, }, { - name: "intersection exists, wrapped", - inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), - inputGVK: []schema.GroupVersionKind{gvk1}, - expected: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), + name: "intersection exists, wrapped", + inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), + inputGVK: []schema.GroupVersionKind{gvk1}, + expectedErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), + expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { - name: "intersection exists, unwrapped", - inputErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, - inputGVK: []schema.GroupVersionKind{gvk1}, - expected: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + name: "intersection exists, unwrapped", + inputErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + inputGVK: []schema.GroupVersionKind{gvk1}, + expectedErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { - name: "intersection does not exist", - inputErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, - inputGVK: []schema.GroupVersionKind{gvk2}, - expected: nil, + name: "intersection does not exist", + inputErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + inputGVK: []schema.GroupVersionKind{gvk2}, + expectedErr: nil, + expectedFailingGVKs: nil, }, { - name: "intersection does not exist, GVKs is empty", - inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), - inputGVK: []schema.GroupVersionKind{}, - expected: nil, + name: "intersection does not exist, GVKs is empty", + inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), + inputGVK: []schema.GroupVersionKind{}, + expectedErr: nil, + expectedFailingGVKs: nil, }, { - name: "global error, gvks inputed", - inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), - inputGVK: []schema.GroupVersionKind{gvk1}, - expected: fmt.Errorf("some err: %w", errors.New("some other err")), + name: "global error, gvks inputed", + inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), + inputGVK: []schema.GroupVersionKind{gvk1}, + expectedErr: fmt.Errorf("some err: %w", errors.New("some other err")), + expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { - name: "global error, no gvks inputed", - inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), - inputGVK: []schema.GroupVersionKind{}, - expected: fmt.Errorf("some err: %w", errors.New("some other err")), + name: "global error, no gvks inputed", + inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), + inputGVK: []schema.GroupVersionKind{}, + expectedErr: fmt.Errorf("some err: %w", errors.New("some other err")), + expectedFailingGVKs: []schema.GroupVersionKind{}, }, { - name: "global unwrapped error, gvks inputed", - inputErr: errors.New("some err"), - inputGVK: []schema.GroupVersionKind{gvk1}, - expected: errors.New("some err"), + name: "global unwrapped error, gvks inputed", + inputErr: errors.New("some err"), + inputGVK: []schema.GroupVersionKind{gvk1}, + expectedErr: errors.New("some err"), + expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { - name: "nested gvk error, intersection", - inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), - inputGVK: []schema.GroupVersionKind{gvk1}, - expected: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + name: "nested gvk error, intersection", + inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + inputGVK: []schema.GroupVersionKind{gvk1}, + expectedErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { - name: "nested gvk error, no intersection", - inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), - inputGVK: []schema.GroupVersionKind{gvk2}, - expected: nil, + name: "nested gvk error, no intersection", + inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + inputGVK: []schema.GroupVersionKind{gvk2}, + expectedErr: nil, + expectedFailingGVKs: nil, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := interpretErr(logger, tc.inputErr, tc.inputGVK) + gvks, gotErr := interpretErr(logger, tc.inputErr, tc.inputGVK) + require.ElementsMatch(t, gvks, tc.expectedFailingGVKs) - if tc.expected != nil { - require.Equal(t, tc.expected.Error(), got.Error()) + if tc.expectedErr != nil { + require.Equal(t, tc.expectedErr.Error(), gotErr.Error()) } else { - require.Nil(t, got, fmt.Sprintf("expected nil error, got: %s", got)) + require.Nil(t, gotErr, fmt.Sprintf("expected nil error, got: %s", gotErr)) } }) } diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 98fe5d17890..3d6e7508155 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -195,15 +195,15 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque r.tracker.DisableStats() } - r.tracker.For(instance.GroupVersionKind()).Observe(instance) - r.cacheManager.ExcludeProcesses(newExcluder) configSourceKey := aggregator.Key{Source: "config", ID: request.NamespacedName.String()} if err := r.cacheManager.UpsertSource(ctx, configSourceKey, gvksToSync); err != nil { + r.tracker.For(instance.GroupVersionKind()).TryCancelExpect(instance) + return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) } - // r.tracker.For(instance.GroupVersionKind()).Observe(instance) + r.tracker.For(instance.GroupVersionKind()).Observe(instance) return reconcile.Result{}, nil } diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index 946dac21d80..8ad6442c25b 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -143,6 +143,8 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ sk := aggregator.Key{Source: "syncset", ID: request.NamespacedName.String()} if err := r.cacheManager.UpsertSource(ctx, sk, gvks); err != nil { + syncsetTr.TryCancelExpect(syncset) + return reconcile.Result{Requeue: true}, fmt.Errorf("synceset-controller: error changing watches: %w", err) } diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index e8e0f93f035..a8aed9b082f 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -228,6 +228,11 @@ func (t *Tracker) CancelData(gvk schema.GroupVersionKind) { t.data.Remove(gvk) } +func (t *Tracker) TryCancelData(gvk schema.GroupVersionKind) { + log.V(logging.DebugLevel).Info("try to cancel tracking for data", "gvk", gvk) + t.data.TryCancel(gvk) +} + // Satisfied returns true if all tracked expectations have been satisfied. func (t *Tracker) Satisfied() bool { // Check circuit-breaker first. Once satisfied, always satisfied. diff --git a/pkg/readiness/ready_tracker_unit_test.go b/pkg/readiness/ready_tracker_unit_test.go index bfdc38fc1e5..7394dc3967c 100644 --- a/pkg/readiness/ready_tracker_unit_test.go +++ b/pkg/readiness/ready_tracker_unit_test.go @@ -17,19 +17,27 @@ package readiness import ( "context" + "fmt" "sync" "testing" + "time" "github.com/onsi/gomega" "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + "github.com/stretchr/testify/require" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" ) -// Stub out the lister. -type dummyLister struct{} +type lister struct { + templates bool // list templates + syncsets bool // list syncsets +} var scheme *runtime.Scheme @@ -38,31 +46,75 @@ func init() { if err := v1beta1.AddToScheme(scheme); err != nil { panic(err) } + if err := syncsetv1alpha1.AddToScheme(scheme); err != nil { + panic(err) + } } -var testConstraintTemplate = templates.ConstraintTemplate{ - ObjectMeta: v1.ObjectMeta{ - Name: "test-contraint-template", - }, - Spec: templates.ConstraintTemplateSpec{ - CRD: templates.CRD{ - Spec: templates.CRDSpec{ - Names: templates.Names{ - Kind: "test-constraint", +var ( + testConstraintTemplate = templates.ConstraintTemplate{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-contraint-template", + }, + Spec: templates.ConstraintTemplateSpec{ + CRD: templates.CRD{ + Spec: templates.CRDSpec{ + Names: templates.Names{ + Kind: "test-constraint", + }, }, }, }, - }, -} + } + + testSyncSet = syncsetv1alpha1.SyncSet{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-sycnset", + }, + Spec: syncsetv1alpha1.SyncSetSpec{ + GVKs: []syncsetv1alpha1.GVKEntry{ + {Group: "", Version: "v1", Kind: "Pod"}, + }, + }, + } + + podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + + testUn = unstructured.Unstructured{} +) -func (dl dummyLister) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - if l, ok := list.(*v1beta1.ConstraintTemplateList); ok { +func (dl lister) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + // failures will be swallowed by readiness.retryAll + switch list := list.(type) { + case *v1beta1.ConstraintTemplateList: + if !dl.templates { + return nil + } i := v1beta1.ConstraintTemplate{} if err := scheme.Convert(&testConstraintTemplate, &i, nil); err != nil { - // These failures will be swallowed by readiness.retryAll return err } - l.Items = []v1beta1.ConstraintTemplate{i} + list.Items = []v1beta1.ConstraintTemplate{i} + case *syncsetv1alpha1.SyncSetList: + if !dl.syncsets { + return nil + } + i := syncsetv1alpha1.SyncSet{} + if err := scheme.Convert(&testSyncSet, &i, nil); err != nil { + return err + } + list.Items = []syncsetv1alpha1.SyncSet{i} + case *unstructured.UnstructuredList: + if !dl.syncsets { + return nil + } + i := unstructured.Unstructured{} + if err := scheme.Convert(&testUn, &i, nil); err != nil { + return err + } + list.Items = []unstructured.Unstructured{i} + default: + return nil } return nil } @@ -71,7 +123,7 @@ func (dl dummyLister) List(ctx context.Context, list client.ObjectList, opts ... func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { g := gomega.NewWithT(t) - l := dummyLister{} + l := lister{templates: true} rt := newTracker(l, false, false, false, func() objData { return objData{retries: 0} }) @@ -113,7 +165,7 @@ func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { g := gomega.NewWithT(t) - l := dummyLister{} + l := lister{templates: true} rt := newTracker(l, false, false, false, func() objData { return objData{retries: 2} }) @@ -162,3 +214,60 @@ func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { t.Fatal("tracker with 0 retries and cancellation should be satisfied") } } + +func Test_Tracker_TryCancelData(t *testing.T) { + l := lister{syncsets: true} + for _, tt := range []struct { + name string + objDataFn func() objData + }{ + {name: "no retries"}, + {name: "with retries", objDataFn: func() objData { + return objData{retries: 2} + }}, + } { + rt := newTracker(l, false, false, false, tt.objDataFn) + + ctx, cancel := context.WithCancel(context.Background()) + var runErr error + runWg := sync.WaitGroup{} + runWg.Add(1) + go func() { + runErr = rt.Run(ctx) + runWg.Done() + }() + + require.Eventually(t, func() bool { + return rt.Populated() + }, 10*time.Second, 1*time.Second, "waiting for RT to populated") + require.False(t, rt.Satisfied(), "tracker with 2 retries should not be satisfied") + + // observe the sync source for readiness + rt.syncsets.Observe(&testSyncSet) + + var retries int + if tt.objDataFn == nil { + retries = 0 + } else { + retries = tt.objDataFn().retries + } + + for i := retries; i > 0; i-- { + require.False(t, rt.data.Satisfied(), "data tracker should not be satisfied") + require.False(t, rt.Satisfied(), fmt.Sprintf("tracker with %d retries should not be satisfied", i)) + rt.TryCancelData(podGVK) + } + + rt.TryCancelData(podGVK) // at this point there should no retries + require.True(t, rt.Satisfied(), "tracker with 0 retries and cancellation should be satisfied") + require.True(t, rt.data.Satisfied(), "data tracker should be satisfied") + + _, removed := rt.data.removed[podGVK] + require.True(t, removed, "expected the podGVK to have been removed") + + // cleanup test + cancel() + runWg.Wait() + require.NoError(t, runErr, "Tracker Run() failed") + } +} diff --git a/pkg/readiness/tracker_map.go b/pkg/readiness/tracker_map.go index 57a596eaae1..fed0ffafec3 100644 --- a/pkg/readiness/tracker_map.go +++ b/pkg/readiness/tracker_map.go @@ -22,17 +22,23 @@ import ( ) type trackerMap struct { - mu sync.RWMutex - m map[schema.GroupVersionKind]*objectTracker - removed map[schema.GroupVersionKind]struct{} - fn objDataFactory + mu sync.RWMutex + m map[schema.GroupVersionKind]*objectTracker + removed map[schema.GroupVersionKind]struct{} + tryCanceled map[schema.GroupVersionKind]objData + fn objDataFactory } func newTrackerMap(fn objDataFactory) *trackerMap { + if fn == nil { + fn = objDataFromFlags + } + return &trackerMap{ - m: make(map[schema.GroupVersionKind]*objectTracker), - removed: make(map[schema.GroupVersionKind]struct{}), - fn: fn, + m: make(map[schema.GroupVersionKind]*objectTracker), + removed: make(map[schema.GroupVersionKind]struct{}), + tryCanceled: make(map[schema.GroupVersionKind]objData), + fn: fn, } } @@ -91,7 +97,14 @@ func (t *trackerMap) Keys() []schema.GroupVersionKind { func (t *trackerMap) Remove(gvk schema.GroupVersionKind) { t.mu.Lock() defer t.mu.Unlock() + + t.removeNoLock(gvk) +} + +func (t *trackerMap) removeNoLock(gvk schema.GroupVersionKind) { delete(t.m, gvk) + delete(t.tryCanceled, gvk) + t.removed[gvk] = struct{}{} } @@ -118,3 +131,24 @@ func (t *trackerMap) Populated() bool { } return true } + +// Populated returns true if all objectTrackers are populated. +func (t *trackerMap) TryCancel(g schema.GroupVersionKind) bool { + t.mu.RLock() + defer t.mu.RUnlock() + + obj, ok := t.tryCanceled[g] + if !ok { + // need to create a record of this TryCancel call + obj = t.fn() + } + + shouldDel := obj.decrementRetries() + t.tryCanceled[g] = obj // set the changed obj back to the map, as the value is not a pointer + + if shouldDel { + t.removeNoLock(g) + } + + return shouldDel +} From 8da842cf9c7026fa5bd4c39c42da8845cbc4f23b Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 9 Oct 2023 23:36:48 +0000 Subject: [PATCH 11/42] fix lock Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/readiness/tracker_map.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/readiness/tracker_map.go b/pkg/readiness/tracker_map.go index fed0ffafec3..eeb9e11e022 100644 --- a/pkg/readiness/tracker_map.go +++ b/pkg/readiness/tracker_map.go @@ -134,8 +134,8 @@ func (t *trackerMap) Populated() bool { // Populated returns true if all objectTrackers are populated. func (t *trackerMap) TryCancel(g schema.GroupVersionKind) bool { - t.mu.RLock() - defer t.mu.RUnlock() + t.mu.Lock() + defer t.mu.Unlock() obj, ok := t.tryCanceled[g] if !ok { From 50f7abd39d147d71630a5819cec883ed13aade81 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 11 Oct 2023 02:41:58 +0000 Subject: [PATCH 12/42] review: rm comments, mgr starts pruner Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- main.go | 9 ++++++++- pkg/cachemanager/cachemanager.go | 17 +++-------------- pkg/controller/config/config_controller.go | 13 ++++++++++--- pkg/controller/syncset/syncset_controller.go | 3 +-- pkg/readiness/pruner/pruner.go | 6 +++--- pkg/readiness/pruner/pruner_test.go | 4 +++- 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index ea7eba82709..b416547c135 100644 --- a/main.go +++ b/main.go @@ -75,6 +75,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" crzap "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" crWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -488,7 +489,13 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle } p := pruner.NewExpecationsPruner(cm, tracker) - go p.Run(ctx) + err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + return p.Run(ctx) + })) + if err != nil { + setupLog.Error(err, "adding expectations pruner to manager") + return err + } opts := controller.Dependencies{ CFClient: client, diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index dd95ad26d07..748c001e601 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -146,10 +146,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet := watch.NewSet() newWatchSet.Add(c.gvksToSync.GVKs()...) - - diff := c.watchedSet.Difference(newWatchSet) - // any resulting stale data expectations are handled async by the ExpectationsMgr. - c.gvksToDeleteFromCache.AddSet(diff) + c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) var innerError error c.watchedSet.Replace(newWatchSet, func() { @@ -163,11 +160,7 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return innerError } -// interpretErr looks at the error e and unwraps it looking to find an error FailingGVKs() in the error chain. -// If found and there is an intersection with the given gvks parameter with the failing gvks, we return the failing -// gvks and an error that is meant to bubble up to controllers. If there is no intersection, we log the error -// so as not to fully swallow any issues. If no error implemeting FailingGVKs() is found, we consider that -// to be a "global error", so we return all gvks passed in and the error. +// interpret whether the err received is of type TODO and whether it has to do with the provided GVKs. func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ([]schema.GroupVersionKind, error) { if e == nil { return nil, nil @@ -180,7 +173,6 @@ func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ( // search for the first error that implements the FailingGVKs() interface if errors.As(e, &f) { - // this error includes failing gvks; let's match against the original list failedGvks := watch.NewSet() failedGvks.Add(f.FailingGVKs()...) gvksSet := watch.NewSet() @@ -188,11 +180,10 @@ func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ( common := failedGvks.Intersection(gvksSet) if common.Size() > 0 { - // then this error is pertinent to the gvks and needs to be returned return common.Items(), e } - // if no intersection, this error is not about the gvks in this request + // this error is not about the gvks in this request // but we still log it for visibility logger.Info("encountered unrelated error when replacing watch set", "error", e) return nil, nil @@ -213,8 +204,6 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke err := c.replaceWatchSet(ctx) if _, interpreted := interpretErr(log, err, []schema.GroupVersionKind{}); interpreted != nil { - // unlike UpsertSource, we cannot TryCancel or Cancel any expectations as these - // GVKs may be required by other sync sources. return fmt.Errorf("error removing watches for source %v: %w", sourceKey, interpreted) } diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 3d6e7508155..b3e67b8538b 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -43,7 +43,14 @@ const ( finalizerName = "finalizers.gatekeeper.sh/config" ) -var log = logf.Log.WithName("controller").WithValues("kind", "Config") +var ( + log = logf.Log.WithName("controller").WithValues("kind", "Config") + configGVK = schema.GroupVersionKind{ + Group: configv1alpha1.GroupVersion.Group, + Version: configv1alpha1.GroupVersion.Version, + Kind: "Config", + } +) type Adder struct { ControllerSwitch *watch.ControllerSwitch @@ -198,12 +205,12 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque r.cacheManager.ExcludeProcesses(newExcluder) configSourceKey := aggregator.Key{Source: "config", ID: request.NamespacedName.String()} if err := r.cacheManager.UpsertSource(ctx, configSourceKey, gvksToSync); err != nil { - r.tracker.For(instance.GroupVersionKind()).TryCancelExpect(instance) + r.tracker.For(configGVK).TryCancelExpect(instance) return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) } - r.tracker.For(instance.GroupVersionKind()).Observe(instance) + r.tracker.For(configGVK).Observe(instance) return reconcile.Result{}, nil } diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index 8ad6442c25b..51c0e52fca1 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -23,8 +23,7 @@ import ( ) const ( - ctrlName = "syncset-controller" - finalizerName = "finalizers.gatekeeper.sh/syncset" + ctrlName = "syncset-controller" ) var ( diff --git a/pkg/readiness/pruner/pruner.go b/pkg/readiness/pruner/pruner.go index 8dd0ac25982..5aa3935e615 100644 --- a/pkg/readiness/pruner/pruner.go +++ b/pkg/readiness/pruner/pruner.go @@ -26,17 +26,17 @@ func NewExpecationsPruner(cm *cachemanager.CacheManager, rt *readiness.Tracker) } } -func (e *ExpectationsPruner) Run(ctx context.Context) { +func (e *ExpectationsPruner) Run(ctx context.Context) error { ticker := time.NewTicker(tickDuration) for { select { case <-ctx.Done(): - return + return nil case <-ticker.C: if e.tracker.Satisfied() { // we're done, there's no need to // further manage the data sync expectations. - return + return nil } if !e.tracker.SyncSourcesSatisfied() { // not yet ready to prune data expectations. diff --git a/pkg/readiness/pruner/pruner_test.go b/pkg/readiness/pruner/pruner_test.go index 116c59b89da..8a22cd14108 100644 --- a/pkg/readiness/pruner/pruner_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -149,7 +149,9 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { require.NoError(t, testutils.ApplyFixtures(tt.fixturesPath, cfg), "applying base fixtures") em, c := setupTest(ctx, t, tt.startControllers) - go em.Run(ctx) + go func() { + _ = em.Run(ctx) + }() // we have to wait on the Tracker to Populate in order to not // have the Deletes below race with the population of expectations. From 7296196f311892d493cee6cf66420bb72adec236 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 11 Oct 2023 03:52:28 +0000 Subject: [PATCH 13/42] review: move interface, IsUniversal Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 15 +++++++------ pkg/cachemanager/cachemanager_test.go | 22 +++++++++++++----- pkg/watch/errorlist.go | 32 ++++++++++++++++++++------- pkg/watch/manager.go | 8 +++---- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 748c001e601..c7341100341 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -160,19 +160,20 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return innerError } -// interpret whether the err received is of type TODO and whether it has to do with the provided GVKs. +// interpret whether the err received is of type WatchesError and whether it has to do with the provided GVKs. func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ([]schema.GroupVersionKind, error) { if e == nil { return nil, nil } - var f interface { - FailingGVKs() []schema.GroupVersionKind - Error() string - } + var f watch.WatchesError - // search for the first error that implements the FailingGVKs() interface + // search the wrapped err tree for an error that implements the WatchesError interface if errors.As(e, &f) { + if f.IsUniversal() { + return gvks, e + } + failedGvks := watch.NewSet() failedGvks.Add(f.FailingGVKs()...) gvksSet := watch.NewSet() @@ -189,7 +190,7 @@ func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ( return nil, nil } - // otherwise, this is a "global error" + // otherwise, this is some other universal error return gvks, e } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 0123c1bcc95..d7bf09554b8 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -652,26 +652,33 @@ func Test_interpretErr(t *testing.T) { expectedFailingGVKs: nil, }, { - name: "global error, gvks inputed", + name: "universal error, gvks inputed", inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), inputGVK: []schema.GroupVersionKind{gvk1}, expectedErr: fmt.Errorf("some err: %w", errors.New("some other err")), expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { - name: "global error, no gvks inputed", + name: "universal error, no gvks inputed", inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), inputGVK: []schema.GroupVersionKind{}, expectedErr: fmt.Errorf("some err: %w", errors.New("some other err")), expectedFailingGVKs: []schema.GroupVersionKind{}, }, { - name: "global unwrapped error, gvks inputed", + name: "universal unwrapped error, gvks inputed", inputErr: errors.New("some err"), inputGVK: []schema.GroupVersionKind{gvk1}, expectedErr: errors.New("some err"), expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, + { + name: "universal error, nested gvks", + inputErr: fmt.Errorf("some err: %w", &gvkError{isUniversal: true, gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + inputGVK: []schema.GroupVersionKind{gvk1, gvk2}, + expectedErr: fmt.Errorf("some err: %w", &gvkError{isUniversal: true, gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + expectedFailingGVKs: []schema.GroupVersionKind{gvk1, gvk2}, + }, { name: "nested gvk error, intersection", inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), @@ -703,8 +710,9 @@ func Test_interpretErr(t *testing.T) { } type gvkError struct { - gvks []schema.GroupVersionKind - err error + gvks []schema.GroupVersionKind + err error + isUniversal bool } func (e *gvkError) Error() string { @@ -715,6 +723,10 @@ func (e *gvkError) FailingGVKs() []schema.GroupVersionKind { return e.gvks } +func (e *gvkError) IsUniversal() bool { + return e.isUniversal +} + func (e *gvkError) Unwrap() error { if e.err != nil { return e.err diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index 4a87a6a9293..4065b3c2a0e 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -22,19 +22,31 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) -// errorList is an error that aggregates multiple errors. -type errorList []GVKError +// errorList is an error that aggregates multiple gvkErrs. +type errorList struct { + errs []gvkErr + isUniversal bool +} + +type WatchesError interface { + // returns gvks for which we had watch errors + FailingGVKs() []schema.GroupVersionKind + // returns true if this error is not specific to the failing gvks + IsUniversal() bool + Error() string +} -type GVKError struct { +// a gvk annotated err. +type gvkErr struct { err error gvk schema.GroupVersionKind } -func (w GVKError) String() string { +func (w gvkErr) String() string { return w.Error() } -func (w GVKError) Error() string { +func (w gvkErr) Error() string { return fmt.Sprintf("error for gvk: %s: %s", w.gvk, w.err.Error()) } @@ -44,7 +56,7 @@ func (e errorList) String() string { func (e errorList) Error() string { var builder strings.Builder - for i, err := range e { + for i, err := range e.errs { if i > 0 { builder.WriteRune('\n') } @@ -54,10 +66,14 @@ func (e errorList) Error() string { } func (e errorList) FailingGVKs() []schema.GroupVersionKind { - gvks := make([]schema.GroupVersionKind, len(e)) - for _, err := range e { + gvks := make([]schema.GroupVersionKind, len(e.errs)) + for _, err := range e.errs { gvks = append(gvks, err.gvk) } return gvks } + +func (e errorList) IsUniversal() bool { + return e.isUniversal +} diff --git a/pkg/watch/manager.go b/pkg/watch/manager.go index a87fdaf0809..a54137a10e0 100644 --- a/pkg/watch/manager.go +++ b/pkg/watch/manager.go @@ -254,7 +254,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { wm.watchedMux.Lock() defer wm.watchedMux.Unlock() - var errlist errorList + errlist := errorList{errs: []gvkErr{}} desired := wm.managedKinds.Get() for gvk := range wm.watchedKinds { @@ -263,7 +263,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { continue } if err := wm.doRemoveWatch(ctx, r, gvk); err != nil { - errlist = append(errlist, GVKError{gvk: gvk, err: fmt.Errorf("removing watch for %+v %w", gvk, err)}) + errlist.errs = append(errlist.errs, gvkErr{gvk: gvk, err: fmt.Errorf("removing watch for %+v %w", gvk, err)}) } } @@ -273,7 +273,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { continue } if err := wm.doAddWatch(ctx, r, gvk); err != nil { - errlist = append(errlist, GVKError{gvk: gvk, err: fmt.Errorf("adding watch for %+v %w", gvk, err)}) + errlist.errs = append(errlist.errs, gvkErr{gvk: gvk, err: fmt.Errorf("adding watch for %+v %w", gvk, err)}) } } @@ -281,7 +281,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { log.Error(err, "while trying to report gvk count metric") } - if errlist != nil { + if len(errlist.errs) > 0 { return errlist } return nil From 6a260a3c30ca2a1ad9009cebcc044868696e4824 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 11 Oct 2023 04:18:11 +0000 Subject: [PATCH 14/42] review: explicitly remove source Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/syncset/syncset_controller.go | 23 +++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index 51c0e52fca1..d818a4f324d 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -130,17 +130,28 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } } + sk := aggregator.Key{Source: "syncset", ID: request.NamespacedName.String()} + + if !exists || !syncset.GetDeletionTimestamp().IsZero() { + log.V(logging.DebugLevel).Info("handling SyncSet delete", "instance", syncset) - gvks := make([]schema.GroupVersionKind, 0) - if exists && syncset.GetDeletionTimestamp().IsZero() { - log.V(logging.DebugLevel).Info("handling SyncSet update", "instance", syncset) + if err := r.cacheManager.RemoveSource(ctx, sk); err != nil { + syncsetTr.TryCancelExpect(syncset) - for _, entry := range syncset.Spec.GVKs { - gvks = append(gvks, schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind}) + return reconcile.Result{}, fmt.Errorf("synceset-controller: error removing source: %w", err) } + + syncsetTr.CancelExpect(syncset) + + return reconcile.Result{}, nil + } + + log.V(logging.DebugLevel).Info("handling SyncSet update", "instance", syncset) + gvks := []schema.GroupVersionKind{} + for _, entry := range syncset.Spec.GVKs { + gvks = append(gvks, schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind}) } - sk := aggregator.Key{Source: "syncset", ID: request.NamespacedName.String()} if err := r.cacheManager.UpsertSource(ctx, sk, gvks); err != nil { syncsetTr.TryCancelExpect(syncset) From 689b25208d2d117c75e22ba2556d846ab9febb17 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 13 Oct 2023 20:58:54 +0000 Subject: [PATCH 15/42] limit name to 63 char Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- config/crd/kustomization.yaml | 6 ++++++ .../gatekeeper/crds/syncset-customresourcedefinition.yaml | 4 ++++ manifest_staging/deploy/gatekeeper.yaml | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index d711fe22595..f09ca63e235 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -50,6 +50,12 @@ patchesJson6902: kind: CustomResourceDefinition name: assignimage.mutations.gatekeeper.sh path: patches/max_name_size.yaml +- target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: syncsets.syncset.gatekeeper.sh + path: patches/max_name_size.yaml patchesStrategicMerge: #- patches/max_name_size_for_modifyset.yaml diff --git a/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml b/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml index aac6d75125b..d239b742ab6 100644 --- a/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml +++ b/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml @@ -28,6 +28,10 @@ spec: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: + properties: + name: + maxLength: 63 + type: string type: object spec: properties: diff --git a/manifest_staging/deploy/gatekeeper.yaml b/manifest_staging/deploy/gatekeeper.yaml index 7c2ebde9194..29dc494db18 100644 --- a/manifest_staging/deploy/gatekeeper.yaml +++ b/manifest_staging/deploy/gatekeeper.yaml @@ -3397,6 +3397,10 @@ spec: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: + properties: + name: + maxLength: 63 + type: string type: object spec: properties: From 4708b4dea4fd9afbb75dffb2999bb66cd371289d Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 13 Oct 2023 23:09:42 +0000 Subject: [PATCH 16/42] review: comments, naming, HasBalidationOperation Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- main.go | 2 +- pkg/cachemanager/cachemanager.go | 8 +- .../syncset/syncset_controller_test.go | 150 ++++++++++-------- pkg/readiness/object_tracker.go | 11 +- pkg/readiness/pruner/pruner.go | 28 ++-- pkg/readiness/ready_tracker.go | 96 ++++++----- pkg/readiness/tracker_map.go | 4 +- 7 files changed, 164 insertions(+), 135 deletions(-) diff --git a/main.go b/main.go index b416547c135..e925b737a92 100644 --- a/main.go +++ b/main.go @@ -488,7 +488,7 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle return err } - p := pruner.NewExpecationsPruner(cm, tracker) + p := pruner.NewExpectationsPruner(cm, tracker) err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { return p.Run(ctx) })) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index c7341100341..ab4df1816d3 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -131,11 +131,11 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // in the manageCache loop. err := c.replaceWatchSet(ctx) - if failedGVKs, interpreted := interpretErr(log, err, newGVKs); interpreted != nil { + if failedGVKs, ie := interpretErr(log, err, newGVKs); ie != nil { for _, g := range failedGVKs { c.tracker.TryCancelData(g) } - return fmt.Errorf("error establishing watches: %w", interpreted) + return fmt.Errorf("error establishing watches: %w", ie) } return nil @@ -204,8 +204,8 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke } err := c.replaceWatchSet(ctx) - if _, interpreted := interpretErr(log, err, []schema.GroupVersionKind{}); interpreted != nil { - return fmt.Errorf("error removing watches for source %v: %w", sourceKey, interpreted) + if _, ie := interpretErr(log, err, []schema.GroupVersionKind{}); ie != nil { + return fmt.Errorf("error removing watches for source %v: %w", sourceKey, ie) } return nil diff --git a/pkg/controller/syncset/syncset_controller_test.go b/pkg/controller/syncset/syncset_controller_test.go index 99bf2c5edc2..9d28075d281 100644 --- a/pkg/controller/syncset/syncset_controller_test.go +++ b/pkg/controller/syncset/syncset_controller_test.go @@ -2,6 +2,7 @@ package syncset import ( "fmt" + "sync" "testing" "time" @@ -26,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -36,18 +38,23 @@ var ( ) const ( - timeout = time.Second * 20 + timeout = time.Second * 10 tick = time.Second * 2 ) // Test_ReconcileSyncSet_wConfigController verifies that SyncSet and Config resources // can get reconciled and their respective specs are added to the data client. func Test_ReconcileSyncSet_wConfigController(t *testing.T) { - require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) - ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() + require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) + + tr := setupTest(ctx, t, false, true) + mgr := *tr.mgr + c := tr.c + cfClient := tr.cfClient + instanceConfig := testutils.ConfigFor([]schema.GroupVersionKind{}) instanceSyncSet1 := &syncsetv1alpha1.SyncSet{ ObjectMeta: metav1.ObjectMeta{ @@ -62,49 +69,15 @@ func Test_ReconcileSyncSet_wConfigController(t *testing.T) { configMap := testutils.UnstructuredFor(configMapGVK, "", "cm1-name") pod := testutils.UnstructuredFor(podGVK, "", "pod1-name") - mgr, wm := testutils.SetupManager(t, cfg) - c := testclient.NewRetryClient(mgr.GetClient()) - - cfClient := &fakes.FakeCfClient{} - cs := watch.NewSwitch() - tracker, err := readiness.SetupTracker(mgr, false, false, false) - if err != nil { - t.Fatal(err) - } - processExcluder := process.Get() - events := make(chan event.GenericEvent, 1024) - syncMetricsCache := syncutil.NewMetricsCache() - w, err := wm.NewRegistrar( - cm.RegistrarName, - events) - require.NoError(t, err) - - cm, err := cm.NewCacheManager(&cm.Config{ - CfClient: cfClient, - SyncMetricsCache: syncMetricsCache, - Tracker: tracker, - ProcessExcluder: processExcluder, - Registrar: w, - Reader: c, - }) - require.NoError(t, err) - go func() { - assert.NoError(t, cm.Start(ctx)) - }() - - rec, err := newReconciler(mgr, cm, cs, tracker) - require.NoError(t, err, "creating sync set reconciler") - require.NoError(t, add(mgr, rec), "adding syncset reconciler to mgr") - // for sync controller - syncAdder := syncc.Adder{CacheManager: cm, Events: events} + syncAdder := syncc.Adder{CacheManager: tr.cacheMgr, Events: tr.events} require.NoError(t, syncAdder.Add(mgr), "adding sync reconciler to mgr") // now for config controller configAdder := config.Adder{ - CacheManager: cm, - ControllerSwitch: cs, - Tracker: tracker, + CacheManager: tr.cacheMgr, + ControllerSwitch: tr.cs, + Tracker: tr.tracker, } require.NoError(t, configAdder.Add(mgr), "adding config reconciler to mgr") @@ -208,8 +181,6 @@ func Test_ReconcileSyncSet_wConfigController(t *testing.T) { } }) } - - cs.Stop() } func expectedCheck(cfClient *fakes.FakeCfClient, expected []schema.GroupVersionKind) func() bool { @@ -223,36 +194,47 @@ func expectedCheck(cfClient *fakes.FakeCfClient, expected []schema.GroupVersionK } } -func Test_ReconcileSyncSet_Reconcile(t *testing.T) { - require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) - - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() +type testResources struct { + mgr *manager.Manager + requests *sync.Map + cacheMgr *cm.CacheManager + c *testclient.RetryClient + wm *watch.Manager + cfClient *fakes.FakeCfClient + events chan event.GenericEvent + cs *watch.ControllerSwitch + tracker *readiness.Tracker +} - instanceSyncSet := &syncsetv1alpha1.SyncSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "syncset", - }, - Spec: syncsetv1alpha1.SyncSetSpec{ - GVKs: []syncsetv1alpha1.GVKEntry{syncsetv1alpha1.GVKEntry(podGVK)}, - }, - } +func setupTest(ctx context.Context, t *testing.T, wrapReconciler bool, useFakeClient bool) testResources { + require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - driver, err := rego.New() - require.NoError(t, err, "unable to set up driver") - - dataClient, err := constraintclient.NewClient(constraintclient.Targets(&target.K8sValidationTarget{}), constraintclient.Driver(driver)) - require.NoError(t, err, "unable to set up data client") + tr := testResources{} + var dataClient cm.CFDataClient + if useFakeClient { + cfClient := &fakes.FakeCfClient{} + dataClient = cfClient + tr.cfClient = cfClient + } else { + driver, err := rego.New() + require.NoError(t, err, "unable to set up driver") + + dataClient, err = constraintclient.NewClient(constraintclient.Targets(&target.K8sValidationTarget{}), constraintclient.Driver(driver)) + require.NoError(t, err, "unable to set up data client") + } cs := watch.NewSwitch() + tr.cs = cs tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) + tr.tracker = tracker processExcluder := process.Get() events := make(chan event.GenericEvent, 1024) + tr.events = events syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( cm.RegistrarName, @@ -265,19 +247,46 @@ func Test_ReconcileSyncSet_Reconcile(t *testing.T) { assert.NoError(t, cm.Start(ctx)) }() + tr.mgr = &mgr + tr.cacheMgr = cm + tr.c = c + tr.wm = wm + rec, err := newReconciler(mgr, cm, cs, tracker) require.NoError(t, err) - recFn, requests := testutils.SetupTestReconcile(rec) - require.NoError(t, add(mgr, recFn)) + if wrapReconciler { + recFn, requests := testutils.SetupTestReconcile(rec) + require.NoError(t, add(mgr, recFn)) + tr.requests = requests + } else { + require.NoError(t, add(mgr, rec)) + } + + return tr +} + +func Test_ReconcileSyncSet_Reconcile(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + tr := setupTest(ctx, t, true, false) + mgr := *tr.mgr + requests := tr.requests + wm := tr.wm + c := tr.c testutils.StartManager(ctx, t, mgr) + instanceSyncSet := &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "syncset", + }, + Spec: syncsetv1alpha1.SyncSetSpec{ + GVKs: []syncsetv1alpha1.GVKEntry{syncsetv1alpha1.GVKEntry(podGVK)}, + }, + } require.NoError(t, c.Create(ctx, instanceSyncSet)) - defer func() { - ctx := context.Background() - require.NoError(t, c.Delete(ctx, instanceSyncSet)) - }() require.Eventually(t, func() bool { _, ok := requests.Load(reconcile.Request{NamespacedName: types.NamespacedName{Name: "syncset"}}) @@ -295,5 +304,10 @@ func Test_ReconcileSyncSet_Reconcile(t *testing.T) { } require.ElementsMatch(t, wantGVKs, gvks) - cs.Stop() + // now delete the sync source and expect no longer watched gvks + require.NoError(t, c.Delete(ctx, instanceSyncSet)) + require.Eventually(t, func() bool { + return len(wm.GetManagedGVK()) == 0 + }, timeout, tick, "check watched gvks are deleted") + require.ElementsMatch(t, []schema.GroupVersionKind{}, wm.GetManagedGVK()) } diff --git a/pkg/readiness/object_tracker.go b/pkg/readiness/object_tracker.go index d7aa0929837..e491a35db8c 100644 --- a/pkg/readiness/object_tracker.go +++ b/pkg/readiness/object_tracker.go @@ -24,6 +24,7 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" syncset1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -258,7 +259,7 @@ func (t *objectTracker) Observe(o runtime.Object) { // Track for future expectation. t.seen[k] = struct{}{} - log.V(1).Info("[readiness] observed data", "gvk", o.GetObjectKind().GroupVersionKind()) + log.V(logging.DebugLevel).Info("[readiness] observed data", "gvk", o.GetObjectKind().GroupVersionKind()) } func (t *objectTracker) Populated() bool { @@ -301,7 +302,7 @@ func (t *objectTracker) Satisfied() bool { if !needMutate { // Read lock to prevent concurrent read/write while logging readiness state. t.mu.RLock() - log.V(1).Info("readiness state", "gvk", t.gvk, "satisfied", fmt.Sprintf("%d/%d", len(t.satisfied), len(t.expect)+len(t.satisfied)), "populated", t.populated) + log.V(logging.DebugLevel).Info("readiness state", "gvk", t.gvk, "satisfied", fmt.Sprintf("%d/%d", len(t.satisfied), len(t.expect)+len(t.satisfied)), "populated", t.populated) t.mu.RUnlock() return false } @@ -321,15 +322,15 @@ func (t *objectTracker) Satisfied() bool { t.satisfied[k] = struct{}{} resolveCount++ } - log.V(1).Info("resolved pre-observations", "gvk", t.gvk, "count", resolveCount) - log.V(1).Info("readiness state", "gvk", t.gvk, "satisfied", fmt.Sprintf("%d/%d", len(t.satisfied), len(t.expect)+len(t.satisfied))) + log.V(logging.DebugLevel).Info("resolved pre-observations", "gvk", t.gvk, "count", resolveCount) + log.V(logging.DebugLevel).Info("readiness state", "gvk", t.gvk, "satisfied", fmt.Sprintf("%d/%d", len(t.satisfied), len(t.expect)+len(t.satisfied))) // All satisfied if: // 1. Expectations have been previously populated // 2. No expectations remain if t.populated && len(t.expect) == 0 { t.allSatisfied = true - log.V(1).Info("all expectations satisfied", "gvk", t.gvk) + log.V(logging.DebugLevel).Info("all expectations satisfied", "gvk", t.gvk) // Circuit-breaker tripped - free tracking memory t.kindsSnapshot = t.kindsNoLock() // Take snapshot as kinds() depends on the maps we're about to clear. diff --git a/pkg/readiness/pruner/pruner.go b/pkg/readiness/pruner/pruner.go index 5aa3935e615..6a5de34fd05 100644 --- a/pkg/readiness/pruner/pruner.go +++ b/pkg/readiness/pruner/pruner.go @@ -11,15 +11,14 @@ import ( const tickDuration = 3 * time.Second -// ExpectationsPruner fires after sync expectations have been satisfied in the ready Tracker and runs -// until the overall Tracker is satisfied. It removes Data expectations for any -// GVKs that are expected in the Tracker but not watched by the CacheManager. +// ExpectationsPruner polls the ReadyTracker and other data sources in Gatekeeper to remove +// un-satisfiable expectations in the RT that would incorrectly block startup. type ExpectationsPruner struct { cacheMgr *cachemanager.CacheManager tracker *readiness.Tracker } -func NewExpecationsPruner(cm *cachemanager.CacheManager, rt *readiness.Tracker) *ExpectationsPruner { +func NewExpectationsPruner(cm *cachemanager.CacheManager, rt *readiness.Tracker) *ExpectationsPruner { return &ExpectationsPruner{ cacheMgr: cm, tracker: rt, @@ -42,15 +41,20 @@ func (e *ExpectationsPruner) Run(ctx context.Context) error { // not yet ready to prune data expectations. break } + e.watchedGVKs() + } + } +} - watchedGVKs := watch.NewSet() - watchedGVKs.Add(e.cacheMgr.WatchedGVKs()...) - expectedGVKs := watch.NewSet() - expectedGVKs.Add(e.tracker.DataGVKs()...) +// watchedGVKs prunes data expectations that are no longer correct based on the up-to-date +// information in the CacheManager. +func (e *ExpectationsPruner) watchedGVKs() { + watchedGVKs := watch.NewSet() + watchedGVKs.Add(e.cacheMgr.WatchedGVKs()...) + expectedGVKs := watch.NewSet() + expectedGVKs.Add(e.tracker.DataGVKs()...) - for _, gvk := range expectedGVKs.Difference(watchedGVKs).Items() { - e.tracker.CancelData(gvk) - } - } + for _, gvk := range expectedGVKs.Difference(watchedGVKs).Items() { + e.tracker.CancelData(gvk) } } diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index a8aed9b082f..19fa39db074 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -285,12 +285,12 @@ func (t *Tracker) Satisfied() bool { return false } - if !t.syncsets.Satisfied() { - log.V(logging.DebugLevel).Info("expectations unsatisfied", "tracker", "syncset") - return false - } - if operations.HasValidationOperations() { + if !t.syncsets.Satisfied() { + log.V(logging.DebugLevel).Info("expectations unsatisfied", "tracker", "syncset") + return false + } + for _, gvk := range t.data.Keys() { if !t.data.Get(gvk).Satisfied() { log.V(logging.DebugLevel).Info("expectations unsatisfied", "tracker", "data", "gvk", gvk) @@ -344,10 +344,11 @@ func (t *Tracker) Run(ctx context.Context) error { grp.Go(func() error { return t.trackConstraintTemplates(gctx) }) - grp.Go(func() error { - return t.trackSyncSources(gctx) - }) } + grp.Go(func() error { + return t.trackSyncSources(gctx) + }) + grp.Go(func() error { t.statsPrinter(ctx) return nil @@ -399,14 +400,19 @@ func (t *Tracker) Populated() bool { } validationPopulated := true if operations.HasValidationOperations() { - validationPopulated = t.templates.Populated() && t.constraints.Populated() && t.data.Populated() + validationPopulated = t.templates.Populated() && t.constraints.Populated() && t.data.Populated() && t.syncsets.Populated() } - return validationPopulated && t.config.Populated() && mutationPopulated && externalDataProviderPopulated && t.syncsets.Populated() + return validationPopulated && t.config.Populated() && mutationPopulated && externalDataProviderPopulated } // Returns whether both the Config and all SyncSet expectations have been Satisfied. func (t *Tracker) SyncSourcesSatisfied() bool { - return t.config.Satisfied() && t.syncsets.Satisfied() + satisfied := t.config.Satisfied() + if operations.HasValidationOperations() { + satisfied = satisfied && t.syncsets.Satisfied() + } + + return satisfied } // collectForObjectTracker identifies objects that are unsatisfied for the provided @@ -498,8 +504,7 @@ func (t *Tracker) collectInvalidExpectations(ctx context.Context) { log.Error(err, "while collecting for the Config tracker") } - sst := t.syncsets - err = t.collectForObjectTracker(ctx, sst, nil, "SyncSet") + err = t.collectForObjectTracker(ctx, t.syncsets, nil, "SyncSet") if err != nil { log.Error(err, "while collecting for the SyncSet tracker") } @@ -508,7 +513,8 @@ func (t *Tracker) collectInvalidExpectations(ctx context.Context) { for _, gvk := range t.constraints.Keys() { // retrieve the expectations for this key es := t.constraints.Get(gvk) - err = t.collectForObjectTracker(ctx, es, nil, fmt.Sprintf("%s/%s", "Constraints", gvk)) + // the GVK of a constraint has the term "constraint" in it already + err = t.collectForObjectTracker(ctx, es, nil, gvk.String()) if err != nil { log.Error(err, "while collecting for the Constraint type", "gvk", gvk) continue @@ -763,43 +769,45 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { } } - syncsets := &syncsetv1alpha1.SyncSetList{} - lister := retryLister(t.lister, retryAll) - if err := lister.List(ctx, syncsets); err != nil { - log.Error(err, "listing syncsets") - } else { - log.V(logging.DebugLevel).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) + if operations.HasValidationOperations() { + syncsets := &syncsetv1alpha1.SyncSetList{} + lister := retryLister(t.lister, retryAll) + if err := lister.List(ctx, syncsets); err != nil { + log.Error(err, "listing syncsets") + } else { + log.V(logging.DebugLevel).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) - for i := range syncsets.Items { - syncset := syncsets.Items[i] + for i := range syncsets.Items { + syncset := syncsets.Items[i] - t.syncsets.Expect(&syncset) - log.V(logging.DebugLevel).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) + t.syncsets.Expect(&syncset) + log.V(logging.DebugLevel).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) - for i := range syncset.Spec.GVKs { - gvk := syncset.Spec.GVKs[i].ToGroupVersionKind() - if _, ok := handled[gvk]; ok { - log.Info("duplicate GVK to sync", "gvk", gvk) - } + for i := range syncset.Spec.GVKs { + gvk := syncset.Spec.GVKs[i].ToGroupVersionKind() + if _, ok := handled[gvk]; ok { + log.Info("duplicate GVK to sync", "gvk", gvk) + } - handled[gvk] = struct{}{} + handled[gvk] = struct{}{} + } } } - } - - // Expect the resource kinds specified in the Config resource and all SyncSet resources. - // We will fail-open (resolve expectations) for GVKs that are unregistered. - for gvk := range handled { - g := gvk - // Set expectations for individual cached resources - dt := t.ForData(g) - go func() { - err := t.trackData(ctx, g, dt) - if err != nil { - log.Error(err, "aborted trackData", "gvk", g) - } - }() + // Expect the resource kinds specified in the Config resource and all SyncSet resources. + // We will fail-open (resolve expectations) for GVKs that are unregistered. + for gvk := range handled { + g := gvk + + // Set expectations for individual cached resources + dt := t.ForData(g) + go func() { + err := t.trackData(ctx, g, dt) + if err != nil { + log.Error(err, "aborted trackData", "gvk", g) + } + }() + } } return nil diff --git a/pkg/readiness/tracker_map.go b/pkg/readiness/tracker_map.go index eeb9e11e022..079f4a486c7 100644 --- a/pkg/readiness/tracker_map.go +++ b/pkg/readiness/tracker_map.go @@ -132,7 +132,9 @@ func (t *trackerMap) Populated() bool { return true } -// Populated returns true if all objectTrackers are populated. +// TryCancel will check the readinessRetries left on this GVK, and remove +// the expectation for its objectTracker if no retries remain. +// Returns True if it stopped tracking a resource kind. func (t *trackerMap) TryCancel(g schema.GroupVersionKind) bool { t.mu.Lock() defer t.mu.Unlock() From 993760577d3da29b5091cd9e0842f951fa4fa538 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 13 Oct 2023 23:14:05 +0000 Subject: [PATCH 17/42] only add syncset controller for validation Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/syncset/syncset_controller.go | 5 +++++ pkg/operations/operations.go | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index d818a4f324d..7384ed80793 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -8,6 +8,7 @@ import ( cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" + "github.com/open-policy-agent/gatekeeper/v3/pkg/operations" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/api/errors" @@ -45,6 +46,10 @@ type Adder struct { // Add creates a new SyncSetController and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func (a *Adder) Add(mgr manager.Manager) error { + if !operations.HasValidationOperations() { + return nil + } + r, err := newReconciler(mgr, a.CacheManager, a.ControllerSwitch, a.Tracker) if err != nil { return err diff --git a/pkg/operations/operations.go b/pkg/operations/operations.go index d51496d2c50..100f19bfcea 100644 --- a/pkg/operations/operations.go +++ b/pkg/operations/operations.go @@ -124,7 +124,8 @@ func AssignedStringList() []string { } // HasValidationOperations returns `true` if there -// are any operations that would require a constraint/template controller. +// are any operations that would require a constraint or template controller +// or a sync controller. func HasValidationOperations() bool { return IsAssigned(Audit) || IsAssigned(Status) || IsAssigned(Webhook) } From 05af5c9ef891c7b8c6009eab335991ff35164916 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 13 Oct 2023 23:57:14 +0000 Subject: [PATCH 18/42] flesh out errorList Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/watch/errorlist.go | 44 +++++++++++++++++++++++++++++++++--------- pkg/watch/manager.go | 8 ++++---- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index 4065b3c2a0e..a691efc63f4 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -16,16 +16,17 @@ limitations under the License. package watch import ( + "errors" "fmt" "strings" "k8s.io/apimachinery/pkg/runtime/schema" ) -// errorList is an error that aggregates multiple gvkErrs. +// errorList is an error that aggregates multiple errors. type errorList struct { - errs []gvkErr - isUniversal bool + errs []error + containsUniversalErr bool } type WatchesError interface { @@ -50,11 +51,11 @@ func (w gvkErr) Error() string { return fmt.Sprintf("error for gvk: %s: %s", w.gvk, w.err.Error()) } -func (e errorList) String() string { +func (e *errorList) String() string { return e.Error() } -func (e errorList) Error() string { +func (e *errorList) Error() string { var builder strings.Builder for i, err := range e.errs { if i > 0 { @@ -65,15 +66,40 @@ func (e errorList) Error() string { return builder.String() } -func (e errorList) FailingGVKs() []schema.GroupVersionKind { +// returns a new errorList type. +func newErrList() *errorList { + return &errorList{ + errs: []error{}, + } +} + +func (e *errorList) FailingGVKs() []schema.GroupVersionKind { gvks := make([]schema.GroupVersionKind, len(e.errs)) for _, err := range e.errs { - gvks = append(gvks, err.gvk) + var gvkErr gvkErr + if errors.As(err, &gvkErr) { + gvks = append(gvks, gvkErr.gvk) + } } return gvks } -func (e errorList) IsUniversal() bool { - return e.isUniversal +func (e *errorList) IsUniversal() bool { + return e.containsUniversalErr +} + +// adds a non gvk specific error to the list. +func (e *errorList) Add(err error) { + e.errs = append(e.errs, err) + e.containsUniversalErr = true +} + +// adds a gvk specific error to the list. +func (e *errorList) AddGVKErr(gvk schema.GroupVersionKind, err error) { + e.errs = append(e.errs, gvkErr{gvk: gvk, err: err}) +} + +func (e *errorList) Size() int { + return len(e.errs) } diff --git a/pkg/watch/manager.go b/pkg/watch/manager.go index a54137a10e0..a6aefb95c61 100644 --- a/pkg/watch/manager.go +++ b/pkg/watch/manager.go @@ -254,7 +254,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { wm.watchedMux.Lock() defer wm.watchedMux.Unlock() - errlist := errorList{errs: []gvkErr{}} + errlist := newErrList() desired := wm.managedKinds.Get() for gvk := range wm.watchedKinds { @@ -263,7 +263,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { continue } if err := wm.doRemoveWatch(ctx, r, gvk); err != nil { - errlist.errs = append(errlist.errs, gvkErr{gvk: gvk, err: fmt.Errorf("removing watch for %+v %w", gvk, err)}) + errlist.AddGVKErr(gvk, fmt.Errorf("removing watch for %+v %w", gvk, err)) } } @@ -273,7 +273,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { continue } if err := wm.doAddWatch(ctx, r, gvk); err != nil { - errlist.errs = append(errlist.errs, gvkErr{gvk: gvk, err: fmt.Errorf("adding watch for %+v %w", gvk, err)}) + errlist.AddGVKErr(gvk, fmt.Errorf("adding watch for %+v %w", gvk, err)) } } @@ -281,7 +281,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { log.Error(err, "while trying to report gvk count metric") } - if len(errlist.errs) > 0 { + if errlist.Size() > 0 { return errlist } return nil From dda78b65c22f04372c00ded46345055b1ef08832 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 16 Oct 2023 19:24:35 +0000 Subject: [PATCH 19/42] TryCancel all gvks if universal Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 29 ++++++++++++++++++--------- pkg/cachemanager/cachemanager_test.go | 10 +++++++-- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index ab4df1816d3..2ef1c41dde5 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -131,8 +131,17 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // in the manageCache loop. err := c.replaceWatchSet(ctx) - if failedGVKs, ie := interpretErr(log, err, newGVKs); ie != nil { - for _, g := range failedGVKs { + if universal, failedGVKs, ie := interpretErr(log, err, newGVKs); ie != nil { + var gvksToTryCancel []schema.GroupVersionKind + if universal { + // if the err is universal, assume all gvks need TryCancel because of some + // WatchManager internal error and we don't want to block readiness. + gvksToTryCancel = c.gvksToSync.GVKs() + } else { + gvksToTryCancel = failedGVKs + } + + for _, g := range gvksToTryCancel { c.tracker.TryCancelData(g) } return fmt.Errorf("error establishing watches: %w", ie) @@ -160,10 +169,10 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return innerError } -// interpret whether the err received is of type WatchesError and whether it has to do with the provided GVKs. -func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ([]schema.GroupVersionKind, error) { +// interpret whether the err received is of type WatchesError and whether it is specific to the provided GVKs. +func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind, error) { if e == nil { - return nil, nil + return false, nil, nil } var f watch.WatchesError @@ -171,7 +180,7 @@ func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ( // search the wrapped err tree for an error that implements the WatchesError interface if errors.As(e, &f) { if f.IsUniversal() { - return gvks, e + return true, gvks, e } failedGvks := watch.NewSet() @@ -181,17 +190,17 @@ func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) ( common := failedGvks.Intersection(gvksSet) if common.Size() > 0 { - return common.Items(), e + return false, common.Items(), e } // this error is not about the gvks in this request // but we still log it for visibility logger.Info("encountered unrelated error when replacing watch set", "error", e) - return nil, nil + return false, nil, nil } // otherwise, this is some other universal error - return gvks, e + return true, gvks, e } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. @@ -204,7 +213,7 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke } err := c.replaceWatchSet(ctx) - if _, ie := interpretErr(log, err, []schema.GroupVersionKind{}); ie != nil { + if _, _, ie := interpretErr(log, err, []schema.GroupVersionKind{}); ie != nil { return fmt.Errorf("error removing watches for source %v: %w", sourceKey, ie) } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index d7bf09554b8..97723e5b6fc 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -617,6 +617,7 @@ func Test_interpretErr(t *testing.T) { inputGVK []schema.GroupVersionKind expectedErr error expectedFailingGVKs []schema.GroupVersionKind + expectUniversal bool }{ { name: "nil err", @@ -657,6 +658,7 @@ func Test_interpretErr(t *testing.T) { inputGVK: []schema.GroupVersionKind{gvk1}, expectedErr: fmt.Errorf("some err: %w", errors.New("some other err")), expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, + expectUniversal: true, }, { name: "universal error, no gvks inputed", @@ -664,6 +666,7 @@ func Test_interpretErr(t *testing.T) { inputGVK: []schema.GroupVersionKind{}, expectedErr: fmt.Errorf("some err: %w", errors.New("some other err")), expectedFailingGVKs: []schema.GroupVersionKind{}, + expectUniversal: true, }, { name: "universal unwrapped error, gvks inputed", @@ -671,6 +674,7 @@ func Test_interpretErr(t *testing.T) { inputGVK: []schema.GroupVersionKind{gvk1}, expectedErr: errors.New("some err"), expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, + expectUniversal: true, }, { name: "universal error, nested gvks", @@ -678,6 +682,7 @@ func Test_interpretErr(t *testing.T) { inputGVK: []schema.GroupVersionKind{gvk1, gvk2}, expectedErr: fmt.Errorf("some err: %w", &gvkError{isUniversal: true, gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), expectedFailingGVKs: []schema.GroupVersionKind{gvk1, gvk2}, + expectUniversal: true, }, { name: "nested gvk error, intersection", @@ -697,9 +702,10 @@ func Test_interpretErr(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - gvks, gotErr := interpretErr(logger, tc.inputErr, tc.inputGVK) - require.ElementsMatch(t, gvks, tc.expectedFailingGVKs) + universal, gvks, gotErr := interpretErr(logger, tc.inputErr, tc.inputGVK) + require.Equal(t, tc.expectUniversal, universal) + require.ElementsMatch(t, gvks, tc.expectedFailingGVKs) if tc.expectedErr != nil { require.Equal(t, tc.expectedErr.Error(), gotErr.Error()) } else { From 5e4809123c856d57952a427edea8a9c93345f12e Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 19 Oct 2023 00:37:35 +0000 Subject: [PATCH 20/42] review feedback Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- apis/config/v1alpha1/config_types.go | 9 + main.go | 6 +- pkg/cachemanager/cachemanager.go | 54 ++-- pkg/cachemanager/cachemanager_test.go | 240 ++++++++---------- pkg/controller/config/config_controller.go | 6 +- pkg/controller/syncset/syncset_controller.go | 13 +- .../syncset/syncset_controller_test.go | 178 ++++--------- pkg/fakes/watcherr.go | 58 +++++ pkg/readiness/pruner/pruner.go | 2 +- pkg/readiness/pruner/pruner_test.go | 3 +- pkg/readiness/ready_tracker.go | 83 +++--- pkg/watch/errorlist.go | 28 +- pkg/watch/errorlist_test.go | 75 ++++++ test/testutils/controller.go | 18 ++ 14 files changed, 405 insertions(+), 368 deletions(-) create mode 100644 pkg/fakes/watcherr.go create mode 100644 pkg/watch/errorlist_test.go diff --git a/apis/config/v1alpha1/config_types.go b/apis/config/v1alpha1/config_types.go index d3a5b7da3d2..372e496c511 100644 --- a/apis/config/v1alpha1/config_types.go +++ b/apis/config/v1alpha1/config_types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" ) // ConfigSpec defines the desired state of Config. @@ -62,6 +63,14 @@ type SyncOnlyEntry struct { Kind string `json:"kind,omitempty"` } +func (e *SyncOnlyEntry) ToGroupVersionKind() schema.GroupVersionKind { + return schema.GroupVersionKind{ + Group: e.Group, + Version: e.Version, + Kind: e.Kind, + } +} + type MatchEntry struct { Processes []string `json:"processes,omitempty"` ExcludedNamespaces []wildcard.Wildcard `json:"excludedNamespaces,omitempty"` diff --git a/main.go b/main.go index e925b737a92..54a0e6b091e 100644 --- a/main.go +++ b/main.go @@ -75,7 +75,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" crzap "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager" crWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -488,10 +487,7 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle return err } - p := pruner.NewExpectationsPruner(cm, tracker) - err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - return p.Run(ctx) - })) + err = mgr.Add(pruner.NewExpectationsPruner(cm, tracker)) if err != nil { setupLog.Error(err, "adding expectations pruner to manager") return err diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 2ef1c41dde5..8e5173378cd 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/go-logr/logr" "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" @@ -131,10 +130,10 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // in the manageCache loop. err := c.replaceWatchSet(ctx) - if universal, failedGVKs, ie := interpretErr(log, err, newGVKs); ie != nil { + if general, failedGVKs := interpretErr(err, newGVKs); len(failedGVKs) > 0 { var gvksToTryCancel []schema.GroupVersionKind - if universal { - // if the err is universal, assume all gvks need TryCancel because of some + if general { + // if the err is general, assume all gvks need TryCancel because of some // WatchManager internal error and we don't want to block readiness. gvksToTryCancel = c.gvksToSync.GVKs() } else { @@ -144,7 +143,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke for _, g := range gvksToTryCancel { c.tracker.TryCancelData(g) } - return fmt.Errorf("error establishing watches: %w", ie) + return fmt.Errorf("error establishing watches: %w", err) } return nil @@ -169,38 +168,31 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return innerError } -// interpret whether the err received is of type WatchesError and whether it is specific to the provided GVKs. -func interpretErr(logger logr.Logger, e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind, error) { +// interpret if the err received is general or whether it is specific to the provided GVKs. +func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind) { if e == nil { - return false, nil, nil + return false, nil } var f watch.WatchesError + if !errors.As(e, &f) || f.HasGeneralErr() { + return true, nil + } - // search the wrapped err tree for an error that implements the WatchesError interface - if errors.As(e, &f) { - if f.IsUniversal() { - return true, gvks, e - } - - failedGvks := watch.NewSet() - failedGvks.Add(f.FailingGVKs()...) - gvksSet := watch.NewSet() - gvksSet.Add(gvks...) - - common := failedGvks.Intersection(gvksSet) - if common.Size() > 0 { - return false, common.Items(), e - } + failedGvks := watch.NewSet() + failedGvks.Add(f.FailingGVKs()...) + sourceGVKSet := watch.NewSet() + sourceGVKSet.Add(gvks...) - // this error is not about the gvks in this request - // but we still log it for visibility - logger.Info("encountered unrelated error when replacing watch set", "error", e) - return false, nil, nil + common := failedGvks.Intersection(sourceGVKSet) + if common.Size() > 0 { + return false, common.Items() } - // otherwise, this is some other universal error - return true, gvks, e + // this error is not about the gvks in this request + // but we still log it for visibility + log.Info("encountered unrelated error when replacing watch set", "error", e) + return false, nil } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. @@ -213,8 +205,8 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke } err := c.replaceWatchSet(ctx) - if _, _, ie := interpretErr(log, err, []schema.GroupVersionKind{}); ie != nil { - return fmt.Errorf("error removing watches for source %v: %w", sourceKey, ie) + if general, _ := interpretErr(err, []schema.GroupVersionKind{}); general { + return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) } return nil diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 97723e5b6fc..a3b15f89aa6 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -29,10 +29,10 @@ import ( var cfg *rest.Config var ( - configMapGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - nsGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} - nonExistentGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "DoesNotExist"} + configMapGVK = schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"} + podGVK = schema.GroupVersionKind{Version: "v1", Kind: "Pod"} + nsGVK = schema.GroupVersionKind{Version: "v1", Kind: "Namespace"} + nonExistentGVK = schema.GroupVersionKind{Version: "v1", Kind: "DoesNotExist"} sourceA = aggregator.Key{Source: "a", ID: "source"} sourceB = aggregator.Key{Source: "b", ID: "source"} @@ -367,113 +367,140 @@ func TestCacheManager_RemoveObject(t *testing.T) { // TestCacheManager_UpsertSource tests that we can modify the gvk aggregator and watched set when adding a new source. func TestCacheManager_UpsertSource(t *testing.T) { - type sourcesAndGvk struct { - source aggregator.Key - gvks []schema.GroupVersionKind - err error + type source struct { + key aggregator.Key + gvks []schema.GroupVersionKind } tcs := []struct { - name string - sourcesAndGvks []sourcesAndGvk - expectedGVKs []schema.GroupVersionKind + name string + sources []source + expectedGVKs []schema.GroupVersionKind }{ { name: "add one source", - sourcesAndGvks: []sourcesAndGvk{ + sources: []source{ { - source: sourceA, - gvks: []schema.GroupVersionKind{configMapGVK}, + key: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, }, }, expectedGVKs: []schema.GroupVersionKind{configMapGVK}, }, { name: "overwrite source", - sourcesAndGvks: []sourcesAndGvk{ + sources: []source{ { - source: sourceA, - gvks: []schema.GroupVersionKind{configMapGVK}, + key: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, }, { - source: sourceA, - gvks: []schema.GroupVersionKind{podGVK}, + key: sourceA, + gvks: []schema.GroupVersionKind{podGVK}, }, }, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { name: "remove source by not specifying any gvk", - sourcesAndGvks: []sourcesAndGvk{ + sources: []source{ { - source: sourceA, - gvks: []schema.GroupVersionKind{configMapGVK}, + key: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, }, { - source: sourceA, - gvks: []schema.GroupVersionKind{}, + key: sourceA, + gvks: []schema.GroupVersionKind{}, }, }, expectedGVKs: []schema.GroupVersionKind{}, }, { - name: "add two disjoing sources", - sourcesAndGvks: []sourcesAndGvk{ + name: "add two disjoint sources", + sources: []source{ { - source: sourceA, - gvks: []schema.GroupVersionKind{configMapGVK}, + key: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, }, { - source: sourceB, - gvks: []schema.GroupVersionKind{podGVK}, + key: sourceB, + gvks: []schema.GroupVersionKind{podGVK}, }, }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, }, { name: "add two sources with fully overlapping gvks", - sourcesAndGvks: []sourcesAndGvk{ + sources: []source{ { - source: sourceA, - gvks: []schema.GroupVersionKind{podGVK}, + key: sourceA, + gvks: []schema.GroupVersionKind{podGVK}, }, { - source: sourceB, - gvks: []schema.GroupVersionKind{podGVK}, + key: sourceB, + gvks: []schema.GroupVersionKind{podGVK}, }, }, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { name: "add two sources with partially overlapping gvks", - sourcesAndGvks: []sourcesAndGvk{ + sources: []source{ { - source: sourceA, - gvks: []schema.GroupVersionKind{configMapGVK, podGVK}, + key: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK, podGVK}, }, { - source: sourceB, - gvks: []schema.GroupVersionKind{podGVK}, + key: sourceB, + gvks: []schema.GroupVersionKind{podGVK}, }, }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + cacheManager, ctx := makeCacheManager(t) + + for _, source := range tc.sources { + require.NoError(t, cacheManager.UpsertSource(ctx, source.key, source.gvks), fmt.Sprintf("while upserting source: %s", source.key)) + } + + require.ElementsMatch(t, cacheManager.watchedSet.Items(), tc.expectedGVKs) + require.ElementsMatch(t, cacheManager.gvksToSync.GVKs(), tc.expectedGVKs) + }) + } +} + +func TestCacheManager_UpsertSource_errorcases(t *testing.T) { + type source struct { + key aggregator.Key + gvks []schema.GroupVersionKind + err error + } + + tcs := []struct { + name string + sources []source + expectedGVKs []schema.GroupVersionKind + }{ { name: "add two sources where one fails to establish all watches", - sourcesAndGvks: []sourcesAndGvk{ + sources: []source{ { - source: sourceA, - gvks: []schema.GroupVersionKind{configMapGVK}, + key: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, }, { - source: sourceB, - gvks: []schema.GroupVersionKind{podGVK, nonExistentGVK}, + key: sourceB, + gvks: []schema.GroupVersionKind{podGVK, nonExistentGVK}, // UpsertSource will err out because of nonExistentGVK err: errors.New("error for gvk: /v1, Kind=DoesNotExist: adding watch for /v1, Kind=DoesNotExist getting informer for kind: /v1, Kind=DoesNotExist no matches for kind \"DoesNotExist\" in version \"v1\""), }, { - source: sourceC, - gvks: []schema.GroupVersionKind{nsGVK}, + key: sourceC, + gvks: []schema.GroupVersionKind{nsGVK}, // without error interpretation, this upsert would fail because we added a // non existent gvk previously. }, @@ -486,11 +513,11 @@ func TestCacheManager_UpsertSource(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cacheManager, ctx := makeCacheManager(t) - for _, sourceAndGVK := range tc.sourcesAndGvks { - if sourceAndGVK.err != nil { - require.ErrorContains(t, cacheManager.UpsertSource(ctx, sourceAndGVK.source, sourceAndGVK.gvks), sourceAndGVK.err.Error(), fmt.Sprintf("while upserting source: %s", sourceAndGVK.source)) + for _, source := range tc.sources { + if source.err != nil { + require.ErrorContains(t, cacheManager.UpsertSource(ctx, source.key, source.gvks), source.err.Error(), fmt.Sprintf("while upserting source: %s", source.key)) } else { - require.NoError(t, cacheManager.UpsertSource(ctx, sourceAndGVK.source, sourceAndGVK.gvks), fmt.Sprintf("while upserting source: %s", sourceAndGVK.source)) + require.NoError(t, cacheManager.UpsertSource(ctx, source.key, source.gvks), fmt.Sprintf("while upserting source: %s", source.key)) } } @@ -607,7 +634,7 @@ func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Uns } func Test_interpretErr(t *testing.T) { - logger := logr.Discard() + log = logr.Discard() gvk1 := schema.GroupVersionKind{Group: "g1", Version: "v1", Kind: "k1"} gvk2 := schema.GroupVersionKind{Group: "g2", Version: "v2", Kind: "k2"} @@ -615,128 +642,69 @@ func Test_interpretErr(t *testing.T) { name string inputErr error inputGVK []schema.GroupVersionKind - expectedErr error expectedFailingGVKs []schema.GroupVersionKind - expectUniversal bool + expectGeneral bool }{ { - name: "nil err", - inputErr: nil, - expectedErr: nil, + name: "nil err", + inputErr: nil, }, { name: "intersection exists, wrapped", - inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), - inputGVK: []schema.GroupVersionKind{gvk1}, - expectedErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), - expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, - }, - { - name: "intersection exists, unwrapped", - inputErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithErr(errors.New("some other err")), fakes.WithGVKs([]schema.GroupVersionKind{gvk1}))), inputGVK: []schema.GroupVersionKind{gvk1}, - expectedErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { name: "intersection does not exist", - inputErr: &gvkError{gvks: []schema.GroupVersionKind{gvk1}}, + inputErr: fakes.WatchesErr(fakes.WithGVKs([]schema.GroupVersionKind{gvk1})), inputGVK: []schema.GroupVersionKind{gvk2}, - expectedErr: nil, expectedFailingGVKs: nil, }, { - name: "intersection does not exist, GVKs is empty", - inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}}), - inputGVK: []schema.GroupVersionKind{}, - expectedErr: nil, - expectedFailingGVKs: nil, + name: "general error, gvks inputed", + inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), + inputGVK: []schema.GroupVersionKind{gvk1}, + expectGeneral: true, }, { - name: "universal error, gvks inputed", - inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), - inputGVK: []schema.GroupVersionKind{gvk1}, - expectedErr: fmt.Errorf("some err: %w", errors.New("some other err")), - expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, - expectUniversal: true, + name: "some other error, no gvks inputed", + inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), + inputGVK: []schema.GroupVersionKind{}, + expectGeneral: true, }, { - name: "universal error, no gvks inputed", - inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), - inputGVK: []schema.GroupVersionKind{}, - expectedErr: fmt.Errorf("some err: %w", errors.New("some other err")), - expectedFailingGVKs: []schema.GroupVersionKind{}, - expectUniversal: true, + name: "some other unwrapped error, gvks inputed", + inputErr: errors.New("some err"), + inputGVK: []schema.GroupVersionKind{gvk1}, + expectGeneral: true, }, { - name: "universal unwrapped error, gvks inputed", - inputErr: errors.New("some err"), - inputGVK: []schema.GroupVersionKind{gvk1}, - expectedErr: errors.New("some err"), - expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, - expectUniversal: true, - }, - { - name: "universal error, nested gvks", - inputErr: fmt.Errorf("some err: %w", &gvkError{isUniversal: true, gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), - inputGVK: []schema.GroupVersionKind{gvk1, gvk2}, - expectedErr: fmt.Errorf("some err: %w", &gvkError{isUniversal: true, gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), - expectedFailingGVKs: []schema.GroupVersionKind{gvk1, gvk2}, - expectUniversal: true, + name: "general error, nested gvks", + inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.GeneralErr(), fakes.WithErr(errors.New("some other err")), fakes.WithGVKs([]schema.GroupVersionKind{gvk1}))), + inputGVK: []schema.GroupVersionKind{gvk1, gvk2}, + expectGeneral: true, }, { name: "nested gvk error, intersection", - inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithErr(errors.New("some other err")), fakes.WithGVKs([]schema.GroupVersionKind{gvk1}))), inputGVK: []schema.GroupVersionKind{gvk1}, - expectedErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { name: "nested gvk error, no intersection", - inputErr: fmt.Errorf("some err: %w", &gvkError{gvks: []schema.GroupVersionKind{gvk1}, err: errors.New("some other err")}), + inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithErr(errors.New("some other err")), fakes.WithGVKs([]schema.GroupVersionKind{gvk1}))), inputGVK: []schema.GroupVersionKind{gvk2}, - expectedErr: nil, expectedFailingGVKs: nil, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - universal, gvks, gotErr := interpretErr(logger, tc.inputErr, tc.inputGVK) + general, gvks := interpretErr(tc.inputErr, tc.inputGVK) - require.Equal(t, tc.expectUniversal, universal) + require.Equal(t, tc.expectGeneral, general) require.ElementsMatch(t, gvks, tc.expectedFailingGVKs) - if tc.expectedErr != nil { - require.Equal(t, tc.expectedErr.Error(), gotErr.Error()) - } else { - require.Nil(t, gotErr, fmt.Sprintf("expected nil error, got: %s", gotErr)) - } }) } } - -type gvkError struct { - gvks []schema.GroupVersionKind - err error - isUniversal bool -} - -func (e *gvkError) Error() string { - return fmt.Sprintf("failing gvks: %v", e.gvks) -} - -func (e *gvkError) FailingGVKs() []schema.GroupVersionKind { - return e.gvks -} - -func (e *gvkError) IsUniversal() bool { - return e.isUniversal -} - -func (e *gvkError) Unwrap() error { - if e.err != nil { - return e.err - } - - return nil -} diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index b3e67b8538b..ed673ee0cb7 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -45,11 +45,7 @@ const ( var ( log = logf.Log.WithName("controller").WithValues("kind", "Config") - configGVK = schema.GroupVersionKind{ - Group: configv1alpha1.GroupVersion.Group, - Version: configv1alpha1.GroupVersion.Version, - Kind: "Config", - } + configGVK = configv1alpha1.GroupVersion.WithKind("Config") ) type Adder struct { diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index 7384ed80793..57bbd917849 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -30,11 +30,7 @@ const ( var ( log = logf.Log.WithName("controller").WithValues("kind", "SyncSet", logging.Process, "syncset_controller") - syncsetGVK = schema.GroupVersionKind{ - Group: syncsetv1alpha1.GroupVersion.Group, - Version: syncsetv1alpha1.GroupVersion.Version, - Kind: "SyncSet", - } + syncsetGVK = syncsetv1alpha1.GroupVersion.WithKind("SyncSet") ) type Adder struct { @@ -116,9 +112,8 @@ type ReconcileSyncSet struct { func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { // Short-circuit if shutting down. if r.cs != nil { - running := r.cs.Enter() defer r.cs.Exit() - if !running { + if !r.cs.Enter() { return reconcile.Result{}, nil } } @@ -154,13 +149,13 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ log.V(logging.DebugLevel).Info("handling SyncSet update", "instance", syncset) gvks := []schema.GroupVersionKind{} for _, entry := range syncset.Spec.GVKs { - gvks = append(gvks, schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind}) + gvks = append(gvks, entry.ToGroupVersionKind()) } if err := r.cacheManager.UpsertSource(ctx, sk, gvks); err != nil { syncsetTr.TryCancelExpect(syncset) - return reconcile.Result{Requeue: true}, fmt.Errorf("synceset-controller: error changing watches: %w", err) + return reconcile.Result{Requeue: true}, fmt.Errorf("synceset-controller: error upserting watches: %w", err) } syncsetTr.Observe(syncset) diff --git a/pkg/controller/syncset/syncset_controller_test.go b/pkg/controller/syncset/syncset_controller_test.go index 9d28075d281..ecbe307918f 100644 --- a/pkg/controller/syncset/syncset_controller_test.go +++ b/pkg/controller/syncset/syncset_controller_test.go @@ -26,15 +26,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var ( - configMapGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - nsGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} - podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + configMapGVK = schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"} + nsGVK = schema.GroupVersionKind{Version: "v1", Kind: "Namespace"} + podGVK = schema.GroupVersionKind{Version: "v1", Kind: "Pod"} ) const ( @@ -50,115 +51,46 @@ func Test_ReconcileSyncSet_wConfigController(t *testing.T) { require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) - tr := setupTest(ctx, t, false, true) - mgr := *tr.mgr - c := tr.c - cfClient := tr.cfClient + tr := setupTest(ctx, t, setupOpts{useFakeClient: true}) - instanceConfig := testutils.ConfigFor([]schema.GroupVersionKind{}) - instanceSyncSet1 := &syncsetv1alpha1.SyncSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "syncset1", - }, - } - instanceSyncSet2 := &syncsetv1alpha1.SyncSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "syncset2", - }, - } configMap := testutils.UnstructuredFor(configMapGVK, "", "cm1-name") pod := testutils.UnstructuredFor(podGVK, "", "pod1-name") - // for sync controller + // installing sync controller is needed for data to actually be populated in the cache syncAdder := syncc.Adder{CacheManager: tr.cacheMgr, Events: tr.events} - require.NoError(t, syncAdder.Add(mgr), "adding sync reconciler to mgr") + require.NoError(t, syncAdder.Add(*tr.mgr), "adding sync reconciler to mgr") - // now for config controller configAdder := config.Adder{ CacheManager: tr.cacheMgr, ControllerSwitch: tr.cs, Tracker: tr.tracker, } - require.NoError(t, configAdder.Add(mgr), "adding config reconciler to mgr") + require.NoError(t, configAdder.Add(*tr.mgr), "adding config reconciler to mgr") - testutils.StartManager(ctx, t, mgr) + testutils.StartManager(ctx, t, *tr.mgr) - require.NoError(t, c.Create(ctx, configMap), fmt.Sprintf("creating ConfigMap %s", "cm1-mame")) - require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", "pod1-name")) + require.NoError(t, tr.c.Create(ctx, configMap), fmt.Sprintf("creating ConfigMap %s", "cm1-mame")) + require.NoError(t, tr.c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", "pod1-name")) tts := []struct { name string - setup func(t *testing.T) - cleanup func(t *testing.T) + syncSources []client.Object expectedGVKs []schema.GroupVersionKind }{ { name: "config and 1 sync", - setup: func(t *testing.T) { - t.Helper() - - instanceConfig := testutils.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}) - instanceSyncSet := &syncsetv1alpha1.SyncSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "syncset1", - }, - Spec: syncsetv1alpha1.SyncSetSpec{ - GVKs: []syncsetv1alpha1.GVKEntry{ - syncsetv1alpha1.GVKEntry(podGVK), - }, - }, - } - - require.NoError(t, c.Create(ctx, instanceConfig)) - require.NoError(t, c.Create(ctx, instanceSyncSet)) - }, - cleanup: func(t *testing.T) { - t.Helper() - - // reset the sync instances - require.NoError(t, c.Delete(ctx, instanceConfig)) - require.NoError(t, c.Delete(ctx, instanceSyncSet1)) + syncSources: []client.Object{ + testutils.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}), + testutils.SyncSetFor("syncset1", []schema.GroupVersionKind{podGVK}), }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK, nsGVK}, }, { name: "config and multiple sync", - setup: func(t *testing.T) { - t.Helper() - - instanceConfig := testutils.ConfigFor([]schema.GroupVersionKind{configMapGVK}) - instanceSyncSet1 = &syncsetv1alpha1.SyncSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "syncset1", - }, - Spec: syncsetv1alpha1.SyncSetSpec{ - GVKs: []syncsetv1alpha1.GVKEntry{ - syncsetv1alpha1.GVKEntry(podGVK), - }, - }, - } - instanceSyncSet2 = &syncsetv1alpha1.SyncSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "syncset2", - }, - Spec: syncsetv1alpha1.SyncSetSpec{ - GVKs: []syncsetv1alpha1.GVKEntry{ - syncsetv1alpha1.GVKEntry(configMapGVK), - }, - }, - } - - require.NoError(t, c.Create(ctx, instanceConfig)) - require.NoError(t, c.Create(ctx, instanceSyncSet1)) - require.NoError(t, c.Create(ctx, instanceSyncSet2)) - }, - cleanup: func(t *testing.T) { - t.Helper() - - // reset the sync instances - require.NoError(t, c.Delete(ctx, instanceConfig)) - require.NoError(t, c.Delete(ctx, instanceSyncSet1)) - require.NoError(t, c.Delete(ctx, instanceSyncSet2)) + syncSources: []client.Object{ + testutils.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}), + testutils.SyncSetFor("syncset1", []schema.GroupVersionKind{podGVK}), + testutils.SyncSetFor("syncset2", []schema.GroupVersionKind{configMapGVK}), }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, }, @@ -166,31 +98,28 @@ func Test_ReconcileSyncSet_wConfigController(t *testing.T) { for _, tt := range tts { t.Run(tt.name, func(t *testing.T) { - if tt.setup != nil { - tt.setup(t) + for _, o := range tt.syncSources { + require.NoError(t, tr.c.Create(ctx, o)) } - assert.Eventually(t, expectedCheck(cfClient, tt.expectedGVKs), timeout, tick) - - if tt.cleanup != nil { - tt.cleanup(t) + assert.Eventually(t, func() bool { + for _, gvk := range tt.expectedGVKs { + if !tr.cfClient.HasGVK(gvk) { + return false + } + } + return true + }, timeout, tick) - require.Eventually(t, func() bool { - return cfClient.Len() == 0 - }, timeout, tick, "could not cleanup") + // reset the sync instances for a clean slate between test cases + for _, o := range tt.syncSources { + require.NoError(t, tr.c.Delete(ctx, o)) } - }) - } -} -func expectedCheck(cfClient *fakes.FakeCfClient, expected []schema.GroupVersionKind) func() bool { - return func() bool { - for _, gvk := range expected { - if !cfClient.HasGVK(gvk) { - return false - } - } - return true + require.Eventually(t, func() bool { + return tr.cfClient.Len() == 0 + }, timeout, tick, "could not cleanup") + }) } } @@ -206,7 +135,12 @@ type testResources struct { tracker *readiness.Tracker } -func setupTest(ctx context.Context, t *testing.T, wrapReconciler bool, useFakeClient bool) testResources { +type setupOpts struct { + wrapReconciler bool + useFakeClient bool +} + +func setupTest(ctx context.Context, t *testing.T, opts setupOpts) testResources { require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) mgr, wm := testutils.SetupManager(t, cfg) @@ -214,7 +148,7 @@ func setupTest(ctx context.Context, t *testing.T, wrapReconciler bool, useFakeCl tr := testResources{} var dataClient cm.CFDataClient - if useFakeClient { + if opts.useFakeClient { cfClient := &fakes.FakeCfClient{} dataClient = cfClient tr.cfClient = cfClient @@ -255,7 +189,7 @@ func setupTest(ctx context.Context, t *testing.T, wrapReconciler bool, useFakeCl rec, err := newReconciler(mgr, cm, cs, tracker) require.NoError(t, err) - if wrapReconciler { + if opts.wrapReconciler { recFn, requests := testutils.SetupTestReconcile(rec) require.NoError(t, add(mgr, recFn)) tr.requests = requests @@ -270,15 +204,11 @@ func Test_ReconcileSyncSet_Reconcile(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() - tr := setupTest(ctx, t, true, false) - mgr := *tr.mgr - requests := tr.requests - wm := tr.wm - c := tr.c + tr := setupTest(ctx, t, setupOpts{wrapReconciler: true}) - testutils.StartManager(ctx, t, mgr) + testutils.StartManager(ctx, t, *tr.mgr) - instanceSyncSet := &syncsetv1alpha1.SyncSet{ + syncset1 := &syncsetv1alpha1.SyncSet{ ObjectMeta: metav1.ObjectMeta{ Name: "syncset", }, @@ -286,28 +216,28 @@ func Test_ReconcileSyncSet_Reconcile(t *testing.T) { GVKs: []syncsetv1alpha1.GVKEntry{syncsetv1alpha1.GVKEntry(podGVK)}, }, } - require.NoError(t, c.Create(ctx, instanceSyncSet)) + require.NoError(t, tr.c.Create(ctx, syncset1)) require.Eventually(t, func() bool { - _, ok := requests.Load(reconcile.Request{NamespacedName: types.NamespacedName{Name: "syncset"}}) + _, ok := tr.requests.Load(reconcile.Request{NamespacedName: types.NamespacedName{Name: "syncset"}}) return ok }, timeout, tick, "waiting on syncset request to be received") require.Eventually(t, func() bool { - return len(wm.GetManagedGVK()) == 1 + return len(tr.wm.GetManagedGVK()) == 1 }, timeout, tick, "check watched gvks are populated") - gvks := wm.GetManagedGVK() + gvks := tr.wm.GetManagedGVK() wantGVKs := []schema.GroupVersionKind{ {Group: "", Version: "v1", Kind: "Pod"}, } require.ElementsMatch(t, wantGVKs, gvks) // now delete the sync source and expect no longer watched gvks - require.NoError(t, c.Delete(ctx, instanceSyncSet)) + require.NoError(t, tr.c.Delete(ctx, syncset1)) require.Eventually(t, func() bool { - return len(wm.GetManagedGVK()) == 0 + return len(tr.wm.GetManagedGVK()) == 0 }, timeout, tick, "check watched gvks are deleted") - require.ElementsMatch(t, []schema.GroupVersionKind{}, wm.GetManagedGVK()) + require.ElementsMatch(t, []schema.GroupVersionKind{}, tr.wm.GetManagedGVK()) } diff --git a/pkg/fakes/watcherr.go b/pkg/fakes/watcherr.go new file mode 100644 index 00000000000..ab4edcccf06 --- /dev/null +++ b/pkg/fakes/watcherr.go @@ -0,0 +1,58 @@ +package fakes + +import ( + "fmt" + + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ watch.WatchesError = &FakeErr{} + +type FakeErr struct { + gvks []schema.GroupVersionKind + err error + generalErr bool +} + +func (e *FakeErr) Error() string { + return fmt.Sprintf("failing gvks: %v", e.gvks) +} + +func (e *FakeErr) FailingGVKs() []schema.GroupVersionKind { + return e.gvks +} + +func (e *FakeErr) HasGeneralErr() bool { + return e.generalErr +} + +type FOpt func(f *FakeErr) + +func WithErr(e error) FOpt { + return func(f *FakeErr) { + f.err = e + } +} + +func WithGVKs(gvks []schema.GroupVersionKind) FOpt { + return func(f *FakeErr) { + f.gvks = gvks + } +} + +func GeneralErr() FOpt { + return func(f *FakeErr) { + f.generalErr = true + } +} + +func WatchesErr(opts ...FOpt) error { + result := &FakeErr{} + + for _, opt := range opts { + opt(result) + } + + return result +} diff --git a/pkg/readiness/pruner/pruner.go b/pkg/readiness/pruner/pruner.go index 6a5de34fd05..40ea28f1d04 100644 --- a/pkg/readiness/pruner/pruner.go +++ b/pkg/readiness/pruner/pruner.go @@ -25,7 +25,7 @@ func NewExpectationsPruner(cm *cachemanager.CacheManager, rt *readiness.Tracker) } } -func (e *ExpectationsPruner) Run(ctx context.Context) error { +func (e *ExpectationsPruner) Start(ctx context.Context) error { ticker := time.NewTicker(tickDuration) for { select { diff --git a/pkg/readiness/pruner/pruner_test.go b/pkg/readiness/pruner/pruner_test.go index 8a22cd14108..96f2f6f08d5 100644 --- a/pkg/readiness/pruner/pruner_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -121,6 +121,7 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { fixturesPath string syncsetsToDelete []string deleteConfig string + // not starting controllers approximates missing events in the informers cache startControllers bool }{ { @@ -150,7 +151,7 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { em, c := setupTest(ctx, t, tt.startControllers) go func() { - _ = em.Run(ctx) + _ = em.Start(ctx) }() // we have to wait on the Tracker to Populate in order to not diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index 19fa39db074..b70e4292a8a 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -291,7 +291,7 @@ func (t *Tracker) Satisfied() bool { return false } - for _, gvk := range t.data.Keys() { + for _, gvk := range t.DataGVKs() { if !t.data.Get(gvk).Satisfied() { log.V(logging.DebugLevel).Info("expectations unsatisfied", "tracker", "data", "gvk", gvk) return false @@ -522,7 +522,7 @@ func (t *Tracker) collectInvalidExpectations(ctx context.Context) { } // collect data expects - for _, gvk := range t.data.Keys() { + for _, gvk := range t.DataGVKs() { // retrieve the expectations for this key es := t.data.Get(gvk) err = t.collectForObjectTracker(ctx, es, nil, fmt.Sprintf("%s/%s", "Data", gvk)) @@ -744,7 +744,7 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { log.V(logging.DebugLevel).Info("syncset expectations populated") }() - handled := make(map[schema.GroupVersionKind]struct{}) + dataGVKs := make(map[schema.GroupVersionKind]struct{}) cfg, err := t.getConfigResource(ctx) if err != nil { @@ -760,54 +760,53 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { log.V(logging.DebugLevel).Info("setting expectations for config", "configCount", 1) for _, entry := range cfg.Spec.Sync.SyncOnly { - handled[schema.GroupVersionKind{ - Group: entry.Group, - Version: entry.Version, - Kind: entry.Kind, - }] = struct{}{} + dataGVKs[entry.ToGroupVersionKind()] = struct{}{} } } } - if operations.HasValidationOperations() { - syncsets := &syncsetv1alpha1.SyncSetList{} - lister := retryLister(t.lister, retryAll) - if err := lister.List(ctx, syncsets); err != nil { - log.Error(err, "listing syncsets") - } else { - log.V(logging.DebugLevel).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) + // Without validation operations, there is no reason to wait for referential data when deciding readiness. + if !operations.HasValidationOperations() { + return nil + } - for i := range syncsets.Items { - syncset := syncsets.Items[i] + syncsets := &syncsetv1alpha1.SyncSetList{} + lister := retryLister(t.lister, retryAll) + if err := lister.List(ctx, syncsets); err != nil { + log.Error(err, "listing syncsets") + } else { + log.V(logging.DebugLevel).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) - t.syncsets.Expect(&syncset) - log.V(logging.DebugLevel).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) + for i := range syncsets.Items { + syncset := syncsets.Items[i] - for i := range syncset.Spec.GVKs { - gvk := syncset.Spec.GVKs[i].ToGroupVersionKind() - if _, ok := handled[gvk]; ok { - log.Info("duplicate GVK to sync", "gvk", gvk) - } + t.syncsets.Expect(&syncset) + log.V(logging.DebugLevel).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) - handled[gvk] = struct{}{} + for i := range syncset.Spec.GVKs { + gvk := syncset.Spec.GVKs[i].ToGroupVersionKind() + if _, ok := dataGVKs[gvk]; ok { + log.Info("duplicate GVK to sync", "gvk", gvk) } + + dataGVKs[gvk] = struct{}{} } } + } - // Expect the resource kinds specified in the Config resource and all SyncSet resources. - // We will fail-open (resolve expectations) for GVKs that are unregistered. - for gvk := range handled { - g := gvk - - // Set expectations for individual cached resources - dt := t.ForData(g) - go func() { - err := t.trackData(ctx, g, dt) - if err != nil { - log.Error(err, "aborted trackData", "gvk", g) - } - }() - } + // Expect the resource kinds specified in the Config resource and all SyncSet resources. + // We will fail-open (resolve expectations) for GVKs that are unregistered. + for gvk := range dataGVKs { + g := gvk + + // Set expectations for individual cached resources + dt := t.ForData(g) + go func() { + err := t.trackData(ctx, g, dt) + if err != nil { + log.Error(err, "aborted trackData", "gvk", g) + } + }() } return nil @@ -947,7 +946,7 @@ func (t *Tracker) statsPrinter(ctx context.Context) { } } - for _, gvk := range t.data.Keys() { + for _, gvk := range t.DataGVKs() { if t.data.Get(gvk).Satisfied() { continue } @@ -997,8 +996,8 @@ func logUnsatisfiedSyncSet(t *Tracker) { } func logUnsatisfiedConfig(t *Tracker) { - if unsat := t.config.unsatisfied(); len(unsat) > 0 { - log.Info("--- Begin unsatisfied config ---", "populated", t.config.Populated(), "count", len(unsat)) + if !t.config.Satisfied() { + log.Info("--- Begin unsatisfied config ---", "populated", t.config.Populated(), "count", 1) log.Info("unsatisfied Config", "name", keys.Config) log.Info("--- End unsatisfied config ---") } diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index a691efc63f4..22c7a72fc44 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -25,15 +25,15 @@ import ( // errorList is an error that aggregates multiple errors. type errorList struct { - errs []error - containsUniversalErr bool + errs []error + hasGeneralErr bool } type WatchesError interface { // returns gvks for which we had watch errors FailingGVKs() []schema.GroupVersionKind // returns true if this error is not specific to the failing gvks - IsUniversal() bool + HasGeneralErr() bool Error() string } @@ -51,6 +51,13 @@ func (w gvkErr) Error() string { return fmt.Sprintf("error for gvk: %s: %s", w.gvk, w.err.Error()) } +// returns a new errorList type. +func newErrList() *errorList { + return &errorList{ + errs: []error{}, + } +} + func (e *errorList) String() string { return e.Error() } @@ -66,15 +73,8 @@ func (e *errorList) Error() string { return builder.String() } -// returns a new errorList type. -func newErrList() *errorList { - return &errorList{ - errs: []error{}, - } -} - func (e *errorList) FailingGVKs() []schema.GroupVersionKind { - gvks := make([]schema.GroupVersionKind, len(e.errs)) + gvks := []schema.GroupVersionKind{} for _, err := range e.errs { var gvkErr gvkErr if errors.As(err, &gvkErr) { @@ -85,14 +85,14 @@ func (e *errorList) FailingGVKs() []schema.GroupVersionKind { return gvks } -func (e *errorList) IsUniversal() bool { - return e.containsUniversalErr +func (e *errorList) HasGeneralErr() bool { + return e.hasGeneralErr } // adds a non gvk specific error to the list. func (e *errorList) Add(err error) { e.errs = append(e.errs, err) - e.containsUniversalErr = true + e.hasGeneralErr = true } // adds a gvk specific error to the list. diff --git a/pkg/watch/errorlist_test.go b/pkg/watch/errorlist_test.go new file mode 100644 index 00000000000..5754977a373 --- /dev/null +++ b/pkg/watch/errorlist_test.go @@ -0,0 +1,75 @@ +package watch + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func Test_WatchesError(t *testing.T) { + someErr := errors.New("some err") + someGVKA := schema.GroupVersionKind{Group: "a", Version: "b", Kind: "c"} + someGVKB := schema.GroupVersionKind{Group: "x", Version: "y", Kind: "z"} + + type errsToAdd struct { + errs []error + gvkErrs []gvkErr + } + + for _, tt := range []struct { + name string + errsToAdd errsToAdd + expectedGVKs []schema.GroupVersionKind + generalErr bool + }{ + { + name: "gvk errors, no global", + errsToAdd: errsToAdd{ + gvkErrs: []gvkErr{ + {err: someErr, gvk: someGVKA}, + {err: someErr, gvk: someGVKB}, + }, + }, + expectedGVKs: []schema.GroupVersionKind{someGVKA, someGVKB}, + generalErr: false, + }, + { + name: "gvk errors and global", + errsToAdd: errsToAdd{ + gvkErrs: []gvkErr{ + {err: someErr, gvk: someGVKA}, + {err: someErr, gvk: someGVKB}, + }, + errs: []error{someErr, someErr}, + }, + expectedGVKs: []schema.GroupVersionKind{someGVKA, someGVKB}, + generalErr: true, + }, + { + name: "just global", + errsToAdd: errsToAdd{ + gvkErrs: []gvkErr{}, + errs: []error{someErr}, + }, + generalErr: true, + }, + { + name: "nothing", + }, + } { + t.Run(tt.name, func(t *testing.T) { + er := errorList{} + for _, gvkErr := range tt.errsToAdd.gvkErrs { + er.AddGVKErr(gvkErr.gvk, gvkErr.err) + } + for _, err := range tt.errsToAdd.errs { + er.Add(err) + } + + require.ElementsMatch(t, tt.expectedGVKs, er.FailingGVKs()) + require.Equal(t, tt.generalErr, er.HasGeneralErr()) + }) + } +} diff --git a/test/testutils/controller.go b/test/testutils/controller.go index 727a974f7bb..ab0f313122c 100644 --- a/test/testutils/controller.go +++ b/test/testutils/controller.go @@ -15,6 +15,7 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" "github.com/open-policy-agent/gatekeeper/v3/apis" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" v1 "k8s.io/api/core/v1" @@ -200,6 +201,23 @@ func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, *sync return fn, &requests } +// SyncSetFor returns a syncset resource with the given name for the requested set of resources. +func SyncSetFor(name string, kinds []schema.GroupVersionKind) *syncsetv1alpha1.SyncSet { + entries := make([]syncsetv1alpha1.GVKEntry, len(kinds)) + for i := range kinds { + entries[i] = syncsetv1alpha1.GVKEntry(kinds[i]) + } + + return &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: syncsetv1alpha1.SyncSetSpec{ + GVKs: entries, + }, + } +} + // ConfigFor returns a config resource that watches the requested set of resources. func ConfigFor(kinds []schema.GroupVersionKind) *configv1alpha1.Config { entries := make([]configv1alpha1.SyncOnlyEntry, len(kinds)) From 24f9237291061e5d782294131bf42177789ad05a Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 26 Oct 2023 00:24:17 +0000 Subject: [PATCH 21/42] review feedback - naming and style - rehome test utils in fake - add a fake lister to be used in pruner tests - fix general err edge case - export ErrorList Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/aggregator/aggregator.go | 4 +- pkg/cachemanager/cachemanager.go | 7 +- pkg/cachemanager/cachemanager_test.go | 130 +++++-------- pkg/controller/syncset/syncset_controller.go | 4 - .../syncset/syncset_controller_test.go | 172 ++++++------------ pkg/fakes/lister.go | 100 ++++++++++ pkg/fakes/sync.go | 83 +++++++++ pkg/fakes/watcherr.go | 44 +---- pkg/readiness/pruner/pruner.go | 10 +- pkg/readiness/pruner/pruner_test.go | 70 +++++-- pkg/readiness/ready_tracker.go | 8 +- pkg/readiness/ready_tracker_unit_test.go | 106 +++-------- pkg/watch/errorlist.go | 40 ++-- pkg/watch/errorlist_test.go | 14 +- pkg/watch/manager.go | 2 +- test/testutils/controller.go | 83 +-------- 16 files changed, 406 insertions(+), 471 deletions(-) create mode 100644 pkg/fakes/lister.go create mode 100644 pkg/fakes/sync.go diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index b36b3cada13..340dea4f875 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -10,9 +10,9 @@ import ( // Key defines a type, identifier tuple to store // in the GVKAggregator. type Key struct { - // Source specifies the type or where this Key comes from. + // Source specifies the type of the source object. Source string - // ID specifies the name of the type that this Key is. + // ID specifies the name instance of the source object. ID string } diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 8e5173378cd..28a73d89120 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -10,6 +10,7 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" @@ -130,7 +131,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // in the manageCache loop. err := c.replaceWatchSet(ctx) - if general, failedGVKs := interpretErr(err, newGVKs); len(failedGVKs) > 0 { + if general, failedGVKs := interpretErr(err, newGVKs); len(failedGVKs) > 0 || general { var gvksToTryCancel []schema.GroupVersionKind if general { // if the err is general, assume all gvks need TryCancel because of some @@ -174,7 +175,7 @@ func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.Group return false, nil } - var f watch.WatchesError + f := watch.NewErrorList() if !errors.As(e, &f) || f.HasGeneralErr() { return true, nil } @@ -191,7 +192,7 @@ func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.Group // this error is not about the gvks in this request // but we still log it for visibility - log.Info("encountered unrelated error when replacing watch set", "error", e) + log.V(logging.DebugLevel).Info("encountered unrelated error when replacing watch set", "error", e) return false, nil } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index a3b15f89aa6..695f90133de 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -6,7 +6,6 @@ import ( "fmt" "testing" - "github.com/go-logr/logr" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" @@ -89,7 +88,7 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { dataClientForTest := func() CFDataClient { cfdc := &fakes.FakeCfClient{} - cm := unstructuredFor(configMapGVK, "config-test-1") + cm := fakes.UnstructuredFor(configMapGVK, "", "config-test-1") _, err := cfdc.AddData(context.Background(), cm) require.NoError(t, err, "adding ConfigMap config-test-1 in cfClient") @@ -365,13 +364,13 @@ func TestCacheManager_RemoveObject(t *testing.T) { } } +type source struct { + key aggregator.Key + gvks []schema.GroupVersionKind +} + // TestCacheManager_UpsertSource tests that we can modify the gvk aggregator and watched set when adding a new source. func TestCacheManager_UpsertSource(t *testing.T) { - type source struct { - key aggregator.Key - gvks []schema.GroupVersionKind - } - tcs := []struct { name string sources []source @@ -402,7 +401,7 @@ func TestCacheManager_UpsertSource(t *testing.T) { expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { - name: "remove source by not specifying any gvk", + name: "remove GVK from a source", sources: []source{ { key: sourceA, @@ -501,8 +500,7 @@ func TestCacheManager_UpsertSource_errorcases(t *testing.T) { { key: sourceC, gvks: []schema.GroupVersionKind{nsGVK}, - // without error interpretation, this upsert would fail because we added a - // non existent gvk previously. + // this call will not error out even though we added a non existent gvk previously to different sync source. }, }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK, nonExistentGVK, nsGVK}, @@ -531,71 +529,68 @@ func TestCacheManager_UpsertSource_errorcases(t *testing.T) { func TestCacheManager_RemoveSource(t *testing.T) { tcs := []struct { name string - seed func(c *CacheManager) + existingSources []source sourcesToRemove []aggregator.Key expectedGVKs []schema.GroupVersionKind }{ { name: "remove disjoint source", - seed: func(c *CacheManager) { - require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) - require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{configMapGVK})) + existingSources: []source{ + {sourceA, []schema.GroupVersionKind{podGVK}}, + {sourceB, []schema.GroupVersionKind{configMapGVK}}, }, sourcesToRemove: []aggregator.Key{sourceB}, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { name: "remove fully overlapping source", - seed: func(c *CacheManager) { - require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) - require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK})) + existingSources: []source{ + {sourceA, []schema.GroupVersionKind{podGVK}}, + {sourceB, []schema.GroupVersionKind{podGVK}}, }, sourcesToRemove: []aggregator.Key{sourceB}, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { name: "remove partially overlapping source", - seed: func(c *CacheManager) { - require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) - require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) + existingSources: []source{ + {sourceA, []schema.GroupVersionKind{podGVK}}, + {sourceB, []schema.GroupVersionKind{podGVK, configMapGVK}}, }, sourcesToRemove: []aggregator.Key{sourceA}, expectedGVKs: []schema.GroupVersionKind{podGVK, configMapGVK}, }, { name: "remove non existing source", - seed: func(c *CacheManager) { - require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) + existingSources: []source{ + {sourceA, []schema.GroupVersionKind{podGVK}}, }, sourcesToRemove: []aggregator.Key{sourceB}, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { - name: "remove source w a non existing gvk", - seed: func(c *CacheManager) { - require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{nonExistentGVK})) + name: "remove source with a non existing gvk", + existingSources: []source{ + {sourceA, []schema.GroupVersionKind{nonExistentGVK}}, }, sourcesToRemove: []aggregator.Key{sourceA}, expectedGVKs: []schema.GroupVersionKind{}, }, { - name: "remove source from a watch set w a non existing gvk", - seed: func(c *CacheManager) { - require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{nonExistentGVK})) - require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK})) + name: "remove source from a watch set with a non existing gvk", + existingSources: []source{ + {sourceA, []schema.GroupVersionKind{nonExistentGVK}}, + {sourceB, []schema.GroupVersionKind{podGVK}}, }, - // without interpreting the error, removing a source that doesn't reference a non existent gvk - // would still error out. sourcesToRemove: []aggregator.Key{sourceB}, expectedGVKs: []schema.GroupVersionKind{nonExistentGVK}, }, { - name: "remove source w non existent gvk from a watch set w a remaining non existing gvk", - seed: func(c *CacheManager) { - require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{nonExistentGVK})) - require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{nonExistentGVK})) + name: "remove source with non existent gvk from a watch set with a remaining non existing gvk", + existingSources: []source{ + {sourceA, []schema.GroupVersionKind{nonExistentGVK}}, + {sourceB, []schema.GroupVersionKind{nonExistentGVK}}, }, - // without interpreting the error, removing a source here would error out. sourcesToRemove: []aggregator.Key{sourceB}, expectedGVKs: []schema.GroupVersionKind{nonExistentGVK}, }, @@ -604,7 +599,10 @@ func TestCacheManager_RemoveSource(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { cm, ctx := makeCacheManager(t) - tc.seed(cm) + // seed the cachemanager internals + for _, s := range tc.existingSources { + require.NoError(t, cm.gvksToSync.Upsert(s.key, s.gvks)) + } for _, source := range tc.sourcesToRemove { require.NoError(t, cm.RemoveSource(ctx, source)) @@ -615,28 +613,10 @@ func TestCacheManager_RemoveSource(t *testing.T) { } } -func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Unstructured { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - u.SetName(name) - u.SetNamespace("default") - if gvk.Kind == "Pod" { - u.Object["spec"] = map[string]interface{}{ - "containers": []map[string]interface{}{ - { - "name": "foo-container", - "image": "foo-image", - }, - }, - } - } - return u -} - func Test_interpretErr(t *testing.T) { - log = logr.Discard() gvk1 := schema.GroupVersionKind{Group: "g1", Version: "v1", Kind: "k1"} gvk2 := schema.GroupVersionKind{Group: "g2", Version: "v2", Kind: "k2"} + someErr := errors.New("some err") cases := []struct { name string @@ -650,53 +630,29 @@ func Test_interpretErr(t *testing.T) { inputErr: nil, }, { - name: "intersection exists, wrapped", - inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithErr(errors.New("some other err")), fakes.WithGVKs([]schema.GroupVersionKind{gvk1}))), + name: "intersection exists", + inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithGVKsErr([]schema.GroupVersionKind{gvk1}, someErr))), inputGVK: []schema.GroupVersionKind{gvk1}, expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { name: "intersection does not exist", - inputErr: fakes.WatchesErr(fakes.WithGVKs([]schema.GroupVersionKind{gvk1})), + inputErr: fakes.WatchesErr(fakes.WithGVKsErr([]schema.GroupVersionKind{gvk1}, someErr)), inputGVK: []schema.GroupVersionKind{gvk2}, expectedFailingGVKs: nil, }, { - name: "general error, gvks inputed", - inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), + name: "non-watchmanager error reports general error with no GVKs", + inputErr: fmt.Errorf("some err: %w", someErr), inputGVK: []schema.GroupVersionKind{gvk1}, expectGeneral: true, }, { - name: "some other error, no gvks inputed", - inputErr: fmt.Errorf("some err: %w", errors.New("some other err")), - inputGVK: []schema.GroupVersionKind{}, - expectGeneral: true, - }, - { - name: "some other unwrapped error, gvks inputed", - inputErr: errors.New("some err"), - inputGVK: []schema.GroupVersionKind{gvk1}, - expectGeneral: true, - }, - { - name: "general error, nested gvks", - inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.GeneralErr(), fakes.WithErr(errors.New("some other err")), fakes.WithGVKs([]schema.GroupVersionKind{gvk1}))), + name: "general error with failing gvks too", + inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithErr(someErr), fakes.WithGVKsErr([]schema.GroupVersionKind{gvk1}, someErr))), inputGVK: []schema.GroupVersionKind{gvk1, gvk2}, expectGeneral: true, }, - { - name: "nested gvk error, intersection", - inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithErr(errors.New("some other err")), fakes.WithGVKs([]schema.GroupVersionKind{gvk1}))), - inputGVK: []schema.GroupVersionKind{gvk1}, - expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, - }, - { - name: "nested gvk error, no intersection", - inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithErr(errors.New("some other err")), fakes.WithGVKs([]schema.GroupVersionKind{gvk1}))), - inputGVK: []schema.GroupVersionKind{gvk2}, - expectedFailingGVKs: nil, - }, } for _, tc := range cases { diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index 57bbd917849..c96eb3753d4 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -137,12 +137,10 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ if err := r.cacheManager.RemoveSource(ctx, sk); err != nil { syncsetTr.TryCancelExpect(syncset) - return reconcile.Result{}, fmt.Errorf("synceset-controller: error removing source: %w", err) } syncsetTr.CancelExpect(syncset) - return reconcile.Result{}, nil } @@ -154,11 +152,9 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ if err := r.cacheManager.UpsertSource(ctx, sk, gvks); err != nil { syncsetTr.TryCancelExpect(syncset) - return reconcile.Result{Requeue: true}, fmt.Errorf("synceset-controller: error upserting watches: %w", err) } syncsetTr.Observe(syncset) - return reconcile.Result{}, nil } diff --git a/pkg/controller/syncset/syncset_controller_test.go b/pkg/controller/syncset/syncset_controller_test.go index ecbe307918f..375d1030052 100644 --- a/pkg/controller/syncset/syncset_controller_test.go +++ b/pkg/controller/syncset/syncset_controller_test.go @@ -2,13 +2,9 @@ package syncset import ( "fmt" - "sync" "testing" "time" - constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" - "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" - syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" @@ -16,20 +12,16 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var ( @@ -51,26 +43,26 @@ func Test_ReconcileSyncSet_wConfigController(t *testing.T) { require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) - tr := setupTest(ctx, t, setupOpts{useFakeClient: true}) + testRes := setupTest(ctx, t) - configMap := testutils.UnstructuredFor(configMapGVK, "", "cm1-name") - pod := testutils.UnstructuredFor(podGVK, "", "pod1-name") + configMap := fakes.UnstructuredFor(configMapGVK, "", "cm1-name") + pod := fakes.UnstructuredFor(podGVK, "", "pod1-name") - // installing sync controller is needed for data to actually be populated in the cache - syncAdder := syncc.Adder{CacheManager: tr.cacheMgr, Events: tr.events} - require.NoError(t, syncAdder.Add(*tr.mgr), "adding sync reconciler to mgr") + // the sync controller populates the cache based on replay events from the watchmanager + syncAdder := syncc.Adder{CacheManager: testRes.cacheMgr, Events: testRes.events} + require.NoError(t, syncAdder.Add(*testRes.mgr), "adding sync reconciler to mgr") configAdder := config.Adder{ - CacheManager: tr.cacheMgr, - ControllerSwitch: tr.cs, - Tracker: tr.tracker, + CacheManager: testRes.cacheMgr, + ControllerSwitch: testRes.cs, + Tracker: testRes.tracker, } - require.NoError(t, configAdder.Add(*tr.mgr), "adding config reconciler to mgr") + require.NoError(t, configAdder.Add(*testRes.mgr), "adding config reconciler to mgr") - testutils.StartManager(ctx, t, *tr.mgr) + testutils.StartManager(ctx, t, *testRes.mgr) - require.NoError(t, tr.c.Create(ctx, configMap), fmt.Sprintf("creating ConfigMap %s", "cm1-mame")) - require.NoError(t, tr.c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", "pod1-name")) + require.NoError(t, testRes.k8sclient.Create(ctx, configMap), fmt.Sprintf("creating ConfigMap %s", "cm1-mame")) + require.NoError(t, testRes.k8sclient.Create(ctx, pod), fmt.Sprintf("creating Pod %s", "pod1-name")) tts := []struct { name string @@ -80,164 +72,104 @@ func Test_ReconcileSyncSet_wConfigController(t *testing.T) { { name: "config and 1 sync", syncSources: []client.Object{ - testutils.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}), - testutils.SyncSetFor("syncset1", []schema.GroupVersionKind{podGVK}), + fakes.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}), + fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{podGVK}), }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK, nsGVK}, }, { - name: "config and multiple sync", + name: "config only", syncSources: []client.Object{ - testutils.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}), - testutils.SyncSetFor("syncset1", []schema.GroupVersionKind{podGVK}), - testutils.SyncSetFor("syncset2", []schema.GroupVersionKind{configMapGVK}), + fakes.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}), }, - expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, + expectedGVKs: []schema.GroupVersionKind{configMapGVK, nsGVK}, + }, + { + name: "syncset only", + syncSources: []client.Object{ + fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{configMapGVK, nsGVK}), + }, + expectedGVKs: []schema.GroupVersionKind{configMapGVK, nsGVK}, }, } for _, tt := range tts { t.Run(tt.name, func(t *testing.T) { for _, o := range tt.syncSources { - require.NoError(t, tr.c.Create(ctx, o)) + require.NoError(t, testRes.k8sclient.Create(ctx, o)) } assert.Eventually(t, func() bool { for _, gvk := range tt.expectedGVKs { - if !tr.cfClient.HasGVK(gvk) { + if !testRes.cfClient.HasGVK(gvk) { return false } } return true }, timeout, tick) - // reset the sync instances for a clean slate between test cases + // empty the cache to not leak state between tests for _, o := range tt.syncSources { - require.NoError(t, tr.c.Delete(ctx, o)) + require.NoError(t, testRes.k8sclient.Delete(ctx, o)) } - require.Eventually(t, func() bool { - return tr.cfClient.Len() == 0 + return testRes.cfClient.Len() == 0 }, timeout, tick, "could not cleanup") }) } } type testResources struct { - mgr *manager.Manager - requests *sync.Map - cacheMgr *cm.CacheManager - c *testclient.RetryClient - wm *watch.Manager - cfClient *fakes.FakeCfClient - events chan event.GenericEvent - cs *watch.ControllerSwitch - tracker *readiness.Tracker -} - -type setupOpts struct { - wrapReconciler bool - useFakeClient bool + mgr *manager.Manager + cacheMgr *cm.CacheManager + k8sclient *testclient.RetryClient + wm *watch.Manager + cfClient *fakes.FakeCfClient + events chan event.GenericEvent + cs *watch.ControllerSwitch + tracker *readiness.Tracker } -func setupTest(ctx context.Context, t *testing.T, opts setupOpts) testResources { +func setupTest(ctx context.Context, t *testing.T) testResources { require.NoError(t, testutils.CreateGatekeeperNamespace(cfg)) mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - tr := testResources{} - var dataClient cm.CFDataClient - if opts.useFakeClient { - cfClient := &fakes.FakeCfClient{} - dataClient = cfClient - tr.cfClient = cfClient - } else { - driver, err := rego.New() - require.NoError(t, err, "unable to set up driver") - - dataClient, err = constraintclient.NewClient(constraintclient.Targets(&target.K8sValidationTarget{}), constraintclient.Driver(driver)) - require.NoError(t, err, "unable to set up data client") - } + testRes := testResources{} cs := watch.NewSwitch() - tr.cs = cs + testRes.cs = cs tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) - tr.tracker = tracker + testRes.tracker = tracker processExcluder := process.Get() events := make(chan event.GenericEvent, 1024) - tr.events = events + testRes.events = events syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( cm.RegistrarName, events) require.NoError(t, err) - cm, err := cm.NewCacheManager(&cm.Config{CfClient: dataClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, Registrar: w, Reader: c}) + cfClient := &fakes.FakeCfClient{} + testRes.cfClient = cfClient + cm, err := cm.NewCacheManager(&cm.Config{CfClient: cfClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, Registrar: w, Reader: c}) require.NoError(t, err) go func() { assert.NoError(t, cm.Start(ctx)) }() - tr.mgr = &mgr - tr.cacheMgr = cm - tr.c = c - tr.wm = wm + testRes.mgr = &mgr + testRes.cacheMgr = cm + testRes.k8sclient = c + testRes.wm = wm rec, err := newReconciler(mgr, cm, cs, tracker) require.NoError(t, err) - if opts.wrapReconciler { - recFn, requests := testutils.SetupTestReconcile(rec) - require.NoError(t, add(mgr, recFn)) - tr.requests = requests - } else { - require.NoError(t, add(mgr, rec)) - } - - return tr -} - -func Test_ReconcileSyncSet_Reconcile(t *testing.T) { - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - - tr := setupTest(ctx, t, setupOpts{wrapReconciler: true}) - - testutils.StartManager(ctx, t, *tr.mgr) - - syncset1 := &syncsetv1alpha1.SyncSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "syncset", - }, - Spec: syncsetv1alpha1.SyncSetSpec{ - GVKs: []syncsetv1alpha1.GVKEntry{syncsetv1alpha1.GVKEntry(podGVK)}, - }, - } - require.NoError(t, tr.c.Create(ctx, syncset1)) - - require.Eventually(t, func() bool { - _, ok := tr.requests.Load(reconcile.Request{NamespacedName: types.NamespacedName{Name: "syncset"}}) + require.NoError(t, add(mgr, rec)) - return ok - }, timeout, tick, "waiting on syncset request to be received") - - require.Eventually(t, func() bool { - return len(tr.wm.GetManagedGVK()) == 1 - }, timeout, tick, "check watched gvks are populated") - - gvks := tr.wm.GetManagedGVK() - wantGVKs := []schema.GroupVersionKind{ - {Group: "", Version: "v1", Kind: "Pod"}, - } - require.ElementsMatch(t, wantGVKs, gvks) - - // now delete the sync source and expect no longer watched gvks - require.NoError(t, tr.c.Delete(ctx, syncset1)) - require.Eventually(t, func() bool { - return len(tr.wm.GetManagedGVK()) == 0 - }, timeout, tick, "check watched gvks are deleted") - require.ElementsMatch(t, []schema.GroupVersionKind{}, tr.wm.GetManagedGVK()) + return testRes } diff --git a/pkg/fakes/lister.go b/pkg/fakes/lister.go new file mode 100644 index 00000000000..856c0c9e82e --- /dev/null +++ b/pkg/fakes/lister.go @@ -0,0 +1,100 @@ +package fakes + +import ( + "context" + + "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" + "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var scheme *runtime.Scheme + +func init() { + scheme = runtime.NewScheme() + if err := v1beta1.AddToScheme(scheme); err != nil { + panic(err) + } + if err := syncsetv1alpha1.AddToScheme(scheme); err != nil { + panic(err) + } +} + +// Fake lister to use for readiness testing. +type TestLister struct { + templatesToList []*templates.ConstraintTemplate + syncSetsToList []*syncsetv1alpha1.SyncSet +} + +type LOpt func(tl *TestLister) + +func WithConstraintTemplates(t []*templates.ConstraintTemplate) LOpt { + return func(tl *TestLister) { + tl.templatesToList = t + } +} + +func WithSyncSets(s []*syncsetv1alpha1.SyncSet) LOpt { + return func(tl *TestLister) { + tl.syncSetsToList = s + } +} + +func NewTestLister(opts ...LOpt) *TestLister { + tl := &TestLister{} + for _, o := range opts { + o(tl) + } + + return tl +} + +// List implements readiness.Lister. +func (tl *TestLister) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + // failures will be swallowed by readiness.retryAll + switch list := list.(type) { + case *v1beta1.ConstraintTemplateList: + if len(tl.templatesToList) == 0 { + return nil + } + + items := []v1beta1.ConstraintTemplate{} + for _, t := range tl.templatesToList { + i := v1beta1.ConstraintTemplate{} + if err := scheme.Convert(t, &i, nil); err != nil { + return err + } + items = append(items, i) + } + list.Items = items + + case *syncsetv1alpha1.SyncSetList: + if len(tl.syncSetsToList) == 0 { + return nil + } + + items := []syncsetv1alpha1.SyncSet{} + for _, t := range tl.syncSetsToList { + i := syncsetv1alpha1.SyncSet{} + if err := scheme.Convert(t, &i, nil); err != nil { + return err + } + items = append(items, i) + } + list.Items = items + + case *unstructured.UnstructuredList: + if len(tl.syncSetsToList) == 0 { + return nil + } + + list.Items = []unstructured.Unstructured{{}} // return one element per list for unstructured. + default: + return nil + } + + return nil +} diff --git a/pkg/fakes/sync.go b/pkg/fakes/sync.go new file mode 100644 index 00000000000..293b92486f9 --- /dev/null +++ b/pkg/fakes/sync.go @@ -0,0 +1,83 @@ +package fakes + +import ( + configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// ConfigFor returns a config resource that watches the requested set of resources. +func ConfigFor(kinds []schema.GroupVersionKind) *configv1alpha1.Config { + entries := make([]configv1alpha1.SyncOnlyEntry, len(kinds)) + for i := range kinds { + entries[i].Group = kinds[i].Group + entries[i].Version = kinds[i].Version + entries[i].Kind = kinds[i].Kind + } + + return &configv1alpha1.Config{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configv1alpha1.GroupVersion.String(), + Kind: "Config", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "config", + Namespace: "gatekeeper-system", + }, + Spec: configv1alpha1.ConfigSpec{ + Sync: configv1alpha1.Sync{ + SyncOnly: entries, + }, + Match: []configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }, + }, + } +} + +// SyncSetFor returns a syncset resource with the given name for the requested set of resources. +func SyncSetFor(name string, kinds []schema.GroupVersionKind) *syncsetv1alpha1.SyncSet { + entries := make([]syncsetv1alpha1.GVKEntry, len(kinds)) + for i := range kinds { + entries[i] = syncsetv1alpha1.GVKEntry(kinds[i]) + } + + return &syncsetv1alpha1.SyncSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: syncsetv1alpha1.SyncSetSpec{ + GVKs: entries, + }, + } +} + +func UnstructuredFor(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + u.SetName(name) + if namespace == "" { + u.SetNamespace("default") + } else { + u.SetNamespace(namespace) + } + + if gvk.Kind == "Pod" { + u.Object["spec"] = map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "foo-container", + "image": "foo-image", + }, + }, + } + } + + return u +} diff --git a/pkg/fakes/watcherr.go b/pkg/fakes/watcherr.go index ab4edcccf06..50d7a23ec33 100644 --- a/pkg/fakes/watcherr.go +++ b/pkg/fakes/watcherr.go @@ -1,54 +1,28 @@ package fakes import ( - "fmt" - "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/runtime/schema" ) -var _ watch.WatchesError = &FakeErr{} - -type FakeErr struct { - gvks []schema.GroupVersionKind - err error - generalErr bool -} - -func (e *FakeErr) Error() string { - return fmt.Sprintf("failing gvks: %v", e.gvks) -} - -func (e *FakeErr) FailingGVKs() []schema.GroupVersionKind { - return e.gvks -} - -func (e *FakeErr) HasGeneralErr() bool { - return e.generalErr -} - -type FOpt func(f *FakeErr) +type FOpt func(f *watch.ErrorList) func WithErr(e error) FOpt { - return func(f *FakeErr) { - f.err = e - } -} - -func WithGVKs(gvks []schema.GroupVersionKind) FOpt { - return func(f *FakeErr) { - f.gvks = gvks + return func(f *watch.ErrorList) { + f.Add(e) } } -func GeneralErr() FOpt { - return func(f *FakeErr) { - f.generalErr = true +func WithGVKsErr(gvks []schema.GroupVersionKind, e error) FOpt { + return func(f *watch.ErrorList) { + for _, gvk := range gvks { + f.AddGVKErr(gvk, e) + } } } func WatchesErr(opts ...FOpt) error { - result := &FakeErr{} + result := watch.NewErrorList() for _, opt := range opts { opt(result) diff --git a/pkg/readiness/pruner/pruner.go b/pkg/readiness/pruner/pruner.go index 40ea28f1d04..1c26f2c4c3f 100644 --- a/pkg/readiness/pruner/pruner.go +++ b/pkg/readiness/pruner/pruner.go @@ -37,18 +37,16 @@ func (e *ExpectationsPruner) Start(ctx context.Context) error { // further manage the data sync expectations. return nil } - if !e.tracker.SyncSourcesSatisfied() { - // not yet ready to prune data expectations. - break + if e.tracker.SyncSourcesSatisfied() { + e.pruneNotWatchedGVKs() } - e.watchedGVKs() } } } -// watchedGVKs prunes data expectations that are no longer correct based on the up-to-date +// pruneNotWatchedGVKs prunes data expectations that are no longer correct based on the up-to-date // information in the CacheManager. -func (e *ExpectationsPruner) watchedGVKs() { +func (e *ExpectationsPruner) pruneNotWatchedGVKs() { watchedGVKs := watch.NewSet() watchedGVKs.Add(e.cacheMgr.WatchedGVKs()...) expectedGVKs := watch.NewSet() diff --git a/pkg/readiness/pruner/pruner_test.go b/pkg/readiness/pruner/pruner_test.go index 96f2f6f08d5..c1a5d60f0f0 100644 --- a/pkg/readiness/pruner/pruner_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" ) const ( @@ -37,26 +38,39 @@ const ( var cfg *rest.Config var ( - syncsetGVK = schema.GroupVersionKind{ - Group: syncsetv1alpha1.GroupVersion.Group, - Version: syncsetv1alpha1.GroupVersion.Version, - Kind: "SyncSet", - } - configGVK = configv1alpha1.GroupVersion.WithKind("Config") + syncsetGVK = syncsetv1alpha1.GroupVersion.WithKind("SyncSet") + configGVK = configv1alpha1.GroupVersion.WithKind("Config") + + configMapGVK = schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"} + podGVK = schema.GroupVersionKind{Version: "v1", Kind: "Pod"} ) func TestMain(m *testing.M) { testutils.StartControlPlane(m, &cfg, 3) } -func setupTest(ctx context.Context, t *testing.T, startControllers bool) (*ExpectationsPruner, client.Client) { +type testOptions struct { + startControllers bool + testLister readiness.Lister +} + +func setupTest(ctx context.Context, t *testing.T, o testOptions) (*ExpectationsPruner, client.Client) { t.Helper() mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - tracker, err := readiness.SetupTrackerNoReadyz(mgr, false, false, false) - require.NoError(t, err, "setting up tracker") + var tracker *readiness.Tracker + var err error + if o.testLister != nil { + tracker = readiness.NewTracker(o.testLister, false, false, false) + require.NoError(t, mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + return tracker.Run(ctx) + })), "adding tracker to manager") + } else { + tracker, err = readiness.SetupTrackerNoReadyz(mgr, false, false, false) + require.NoError(t, err, "setting up tracker") + } events := make(chan event.GenericEvent, 1024) reg, err := wm.NewRegistrar( @@ -79,7 +93,7 @@ func setupTest(ctx context.Context, t *testing.T, startControllers bool) (*Expec cm, err := cachemanager.NewCacheManager(config) require.NoError(t, err, "creating cachemanager") - if !startControllers { + if !o.startControllers { // need to start the cachemanager if controllers are not started // since the cachemanager is started in the controllers code. require.NoError(t, mgr.Add(cm), "adding cachemanager as a runnable") @@ -149,13 +163,8 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { require.NoError(t, testutils.ApplyFixtures(tt.fixturesPath, cfg), "applying base fixtures") - em, c := setupTest(ctx, t, tt.startControllers) - go func() { - _ = em.Start(ctx) - }() + em, c := setupTest(ctx, t, testOptions{startControllers: tt.startControllers}) - // we have to wait on the Tracker to Populate in order to not - // have the Deletes below race with the population of expectations. require.Eventually(t, func() bool { return em.tracker.Populated() }, timeout, tick, "waiting on tracker to populate") @@ -176,6 +185,8 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { require.NoError(t, c.Delete(ctx, u), fmt.Sprintf("deleting config %s", tt.deleteConfig)) } + em.pruneNotWatchedGVKs() + require.Eventually(t, func() bool { return em.tracker.Satisfied() }, timeout, tick, "waiting on tracker to get satisfied") @@ -184,3 +195,30 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { }) } } + +// Test_ExpectationsMgr_missedInformers verifies that the pruner can handle a scenario +// where the readiness tracker's state will never match the informer cache events. +func Test_ExpectationsMgr_missedInformers(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) + + // because we will use a separate lister for the tracker from the mgr client + // the contents of the readiness tracker will be superset of the contents of the mgr's client + tl := fakes.NewTestLister( + fakes.WithSyncSets([]*syncsetv1alpha1.SyncSet{ + fakes.SyncSetFor("syncset-1", []schema.GroupVersionKind{podGVK, configMapGVK}), + }), + ) + em, _ := setupTest(ctx, t, testOptions{testLister: tl}) + + require.Eventually(t, func() bool { + return em.tracker.SyncSourcesSatisfied() + }, timeout, tick, "waiting on sync sources to get satisfied") + + em.pruneNotWatchedGVKs() + + require.Eventually(t, func() bool { + return em.tracker.Satisfied() + }, timeout, tick, "waiting on tracker to get satisfied") + + cancelFunc() +} diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index b70e4292a8a..5667ff133be 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -797,14 +797,14 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { // Expect the resource kinds specified in the Config resource and all SyncSet resources. // We will fail-open (resolve expectations) for GVKs that are unregistered. for gvk := range dataGVKs { - g := gvk + gvkCpy := gvk // Set expectations for individual cached resources - dt := t.ForData(g) + dt := t.ForData(gvkCpy) go func() { - err := t.trackData(ctx, g, dt) + err := t.trackData(ctx, gvkCpy, dt) if err != nil { - log.Error(err, "aborted trackData", "gvk", g) + log.Error(err, "aborted trackData", "gvk", gvkCpy) } }() } diff --git a/pkg/readiness/ready_tracker_unit_test.go b/pkg/readiness/ready_tracker_unit_test.go index 7394dc3967c..90746fc533c 100644 --- a/pkg/readiness/ready_tracker_unit_test.go +++ b/pkg/readiness/ready_tracker_unit_test.go @@ -23,34 +23,14 @@ import ( "time" "github.com/onsi/gomega" - "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/stretchr/testify/require" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client" ) -type lister struct { - templates bool // list templates - syncsets bool // list syncsets -} - -var scheme *runtime.Scheme - -func init() { - scheme = runtime.NewScheme() - if err := v1beta1.AddToScheme(scheme); err != nil { - panic(err) - } - if err := syncsetv1alpha1.AddToScheme(scheme); err != nil { - panic(err) - } -} - var ( testConstraintTemplate = templates.ConstraintTemplate{ ObjectMeta: v1.ObjectMeta{ @@ -79,51 +59,15 @@ var ( } podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - - testUn = unstructured.Unstructured{} ) -func (dl lister) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - // failures will be swallowed by readiness.retryAll - switch list := list.(type) { - case *v1beta1.ConstraintTemplateList: - if !dl.templates { - return nil - } - i := v1beta1.ConstraintTemplate{} - if err := scheme.Convert(&testConstraintTemplate, &i, nil); err != nil { - return err - } - list.Items = []v1beta1.ConstraintTemplate{i} - case *syncsetv1alpha1.SyncSetList: - if !dl.syncsets { - return nil - } - i := syncsetv1alpha1.SyncSet{} - if err := scheme.Convert(&testSyncSet, &i, nil); err != nil { - return err - } - list.Items = []syncsetv1alpha1.SyncSet{i} - case *unstructured.UnstructuredList: - if !dl.syncsets { - return nil - } - i := unstructured.Unstructured{} - if err := scheme.Convert(&testUn, &i, nil); err != nil { - return err - } - list.Items = []unstructured.Unstructured{i} - default: - return nil - } - return nil -} - // Verify that TryCancelTemplate functions the same as regular CancelTemplate if readinessRetries is set to 0. func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { g := gomega.NewWithT(t) - l := lister{templates: true} + l := fakes.NewTestLister( + fakes.WithConstraintTemplates([]*templates.ConstraintTemplate{&testConstraintTemplate}), + ) rt := newTracker(l, false, false, false, func() objData { return objData{retries: 0} }) @@ -165,7 +109,9 @@ func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { g := gomega.NewWithT(t) - l := lister{templates: true} + l := fakes.NewTestLister( + fakes.WithConstraintTemplates([]*templates.ConstraintTemplate{&testConstraintTemplate}), + ) rt := newTracker(l, false, false, false, func() objData { return objData{retries: 2} }) @@ -216,17 +162,22 @@ func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { } func Test_Tracker_TryCancelData(t *testing.T) { - l := lister{syncsets: true} - for _, tt := range []struct { - name string - objDataFn func() objData + tcs := []struct { + name string + retries int }{ - {name: "no retries"}, - {name: "with retries", objDataFn: func() objData { - return objData{retries: 2} - }}, - } { - rt := newTracker(l, false, false, false, tt.objDataFn) + {name: "no retries", retries: 0}, + {name: "with retries", retries: 2}, + } + + l := fakes.NewTestLister( + fakes.WithSyncSets([]*syncsetv1alpha1.SyncSet{&testSyncSet}), + ) + for _, tt := range tcs { + objDataFn := func() objData { + return objData{retries: tt.retries} + } + rt := newTracker(l, false, false, false, objDataFn) ctx, cancel := context.WithCancel(context.Background()) var runErr error @@ -234,29 +185,24 @@ func Test_Tracker_TryCancelData(t *testing.T) { runWg.Add(1) go func() { runErr = rt.Run(ctx) + // wait for the ready tracker to stop so we don't leak state between tests. runWg.Done() }() require.Eventually(t, func() bool { return rt.Populated() }, 10*time.Second, 1*time.Second, "waiting for RT to populated") - require.False(t, rt.Satisfied(), "tracker with 2 retries should not be satisfied") + require.False(t, rt.Satisfied(), "tracker with retries should not be satisfied") // observe the sync source for readiness rt.syncsets.Observe(&testSyncSet) - var retries int - if tt.objDataFn == nil { - retries = 0 - } else { - retries = tt.objDataFn().retries - } - - for i := retries; i > 0; i-- { + for i := tt.retries; i > 0; i-- { require.False(t, rt.data.Satisfied(), "data tracker should not be satisfied") require.False(t, rt.Satisfied(), fmt.Sprintf("tracker with %d retries should not be satisfied", i)) rt.TryCancelData(podGVK) } + require.False(t, rt.Satisfied(), "tracker should not be satisfied") rt.TryCancelData(podGVK) // at this point there should no retries require.True(t, rt.Satisfied(), "tracker with 0 retries and cancellation should be satisfied") diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index 22c7a72fc44..3a37d4d9525 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -23,21 +23,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) -// errorList is an error that aggregates multiple errors. -type errorList struct { - errs []error - hasGeneralErr bool -} - -type WatchesError interface { - // returns gvks for which we had watch errors - FailingGVKs() []schema.GroupVersionKind - // returns true if this error is not specific to the failing gvks - HasGeneralErr() bool - Error() string -} - -// a gvk annotated err. type gvkErr struct { err error gvk schema.GroupVersionKind @@ -51,18 +36,23 @@ func (w gvkErr) Error() string { return fmt.Sprintf("error for gvk: %s: %s", w.gvk, w.err.Error()) } -// returns a new errorList type. -func newErrList() *errorList { - return &errorList{ +// ErrorList is an error that aggregates multiple errors. +type ErrorList struct { + errs []error + hasGeneralErr bool +} + +func NewErrorList() *ErrorList { + return &ErrorList{ errs: []error{}, } } -func (e *errorList) String() string { +func (e *ErrorList) String() string { return e.Error() } -func (e *errorList) Error() string { +func (e *ErrorList) Error() string { var builder strings.Builder for i, err := range e.errs { if i > 0 { @@ -73,7 +63,7 @@ func (e *errorList) Error() string { return builder.String() } -func (e *errorList) FailingGVKs() []schema.GroupVersionKind { +func (e *ErrorList) FailingGVKs() []schema.GroupVersionKind { gvks := []schema.GroupVersionKind{} for _, err := range e.errs { var gvkErr gvkErr @@ -85,21 +75,21 @@ func (e *errorList) FailingGVKs() []schema.GroupVersionKind { return gvks } -func (e *errorList) HasGeneralErr() bool { +func (e *ErrorList) HasGeneralErr() bool { return e.hasGeneralErr } // adds a non gvk specific error to the list. -func (e *errorList) Add(err error) { +func (e *ErrorList) Add(err error) { e.errs = append(e.errs, err) e.hasGeneralErr = true } // adds a gvk specific error to the list. -func (e *errorList) AddGVKErr(gvk schema.GroupVersionKind, err error) { +func (e *ErrorList) AddGVKErr(gvk schema.GroupVersionKind, err error) { e.errs = append(e.errs, gvkErr{gvk: gvk, err: err}) } -func (e *errorList) Size() int { +func (e *ErrorList) Size() int { return len(e.errs) } diff --git a/pkg/watch/errorlist_test.go b/pkg/watch/errorlist_test.go index 5754977a373..6f8b783cc8d 100644 --- a/pkg/watch/errorlist_test.go +++ b/pkg/watch/errorlist_test.go @@ -18,7 +18,7 @@ func Test_WatchesError(t *testing.T) { gvkErrs []gvkErr } - for _, tt := range []struct { + tcs := []struct { name string errsToAdd errsToAdd expectedGVKs []schema.GroupVersionKind @@ -50,17 +50,15 @@ func Test_WatchesError(t *testing.T) { { name: "just global", errsToAdd: errsToAdd{ - gvkErrs: []gvkErr{}, - errs: []error{someErr}, + errs: []error{someErr}, }, generalErr: true, }, - { - name: "nothing", - }, - } { + } + + for _, tt := range tcs { t.Run(tt.name, func(t *testing.T) { - er := errorList{} + er := NewErrorList() for _, gvkErr := range tt.errsToAdd.gvkErrs { er.AddGVKErr(gvkErr.gvk, gvkErr.err) } diff --git a/pkg/watch/manager.go b/pkg/watch/manager.go index a6aefb95c61..c1e133c8947 100644 --- a/pkg/watch/manager.go +++ b/pkg/watch/manager.go @@ -254,7 +254,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { wm.watchedMux.Lock() defer wm.watchedMux.Unlock() - errlist := newErrList() + errlist := NewErrorList() desired := wm.managedKinds.Get() for gvk := range wm.watchedKinds { diff --git a/test/testutils/controller.go b/test/testutils/controller.go index ab0f313122c..0daa2b0e479 100644 --- a/test/testutils/controller.go +++ b/test/testutils/controller.go @@ -14,15 +14,11 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" "github.com/open-policy-agent/gatekeeper/v3/apis" - configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" - syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" - "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -94,7 +90,7 @@ func DeleteObjectAndConfirm(ctx context.Context, t *testing.T, c client.Client, t.Helper() // Construct a single-use Unstructured to send the Delete request. - toDelete := UnstructuredFor(gvk, namespace, name) + toDelete := fakes.UnstructuredFor(gvk, namespace, name) err := c.Delete(ctx, toDelete) if apierrors.IsNotFound(err) { return @@ -107,7 +103,7 @@ func DeleteObjectAndConfirm(ctx context.Context, t *testing.T, c client.Client, }, func() error { // Construct a single-use Unstructured to send the Get request. It isn't // safe to reuse Unstructureds for each retry as Get modifies its input. - toGet := UnstructuredFor(gvk, namespace, name) + toGet := fakes.UnstructuredFor(gvk, namespace, name) key := client.ObjectKey{Namespace: namespace, Name: name} err2 := c.Get(ctx, key, toGet) if apierrors.IsGone(err2) || apierrors.IsNotFound(err2) { @@ -200,76 +196,3 @@ func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, *sync }) return fn, &requests } - -// SyncSetFor returns a syncset resource with the given name for the requested set of resources. -func SyncSetFor(name string, kinds []schema.GroupVersionKind) *syncsetv1alpha1.SyncSet { - entries := make([]syncsetv1alpha1.GVKEntry, len(kinds)) - for i := range kinds { - entries[i] = syncsetv1alpha1.GVKEntry(kinds[i]) - } - - return &syncsetv1alpha1.SyncSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: syncsetv1alpha1.SyncSetSpec{ - GVKs: entries, - }, - } -} - -// ConfigFor returns a config resource that watches the requested set of resources. -func ConfigFor(kinds []schema.GroupVersionKind) *configv1alpha1.Config { - entries := make([]configv1alpha1.SyncOnlyEntry, len(kinds)) - for i := range kinds { - entries[i].Group = kinds[i].Group - entries[i].Version = kinds[i].Version - entries[i].Kind = kinds[i].Kind - } - - return &configv1alpha1.Config{ - TypeMeta: metav1.TypeMeta{ - APIVersion: configv1alpha1.GroupVersion.String(), - Kind: "Config", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "config", - Namespace: "gatekeeper-system", - }, - Spec: configv1alpha1.ConfigSpec{ - Sync: configv1alpha1.Sync{ - SyncOnly: entries, - }, - Match: []configv1alpha1.MatchEntry{ - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }, - }, - } -} - -func UnstructuredFor(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - u.SetName(name) - if namespace == "" { - u.SetNamespace("default") - } else { - u.SetNamespace(namespace) - } - - if gvk.Kind == "Pod" { - u.Object["spec"] = map[string]interface{}{ - "containers": []map[string]interface{}{ - { - "name": "foo-container", - "image": "foo-image", - }, - }, - } - } - - return u -} From 53a7625fd9d4c37ada32abf3db5612ad8b2f3d08 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Sat, 28 Oct 2023 21:06:09 +0000 Subject: [PATCH 22/42] review feedback - test setup nits - comments - move fns Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/aggregator/aggregator.go | 2 +- pkg/cachemanager/cachemanager_test.go | 20 ++++--- pkg/fakes/fakecfdataclient.go | 25 +++++++++ pkg/fakes/sync.go | 25 --------- pkg/fakes/watcherr.go | 32 ------------ pkg/readiness/pruner/pruner_test.go | 64 +++++++++++++++-------- pkg/readiness/ready_tracker_unit_test.go | 6 +-- pkg/watch/errorlist_test.go | 12 ++--- 8 files changed, 91 insertions(+), 95 deletions(-) delete mode 100644 pkg/fakes/watcherr.go diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index 340dea4f875..ee7747fad39 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -12,7 +12,7 @@ import ( type Key struct { // Source specifies the type of the source object. Source string - // ID specifies the name instance of the source object. + // ID specifies the name of the instance of the source object. ID string } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 695f90133de..cae59edb008 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -500,7 +500,9 @@ func TestCacheManager_UpsertSource_errorcases(t *testing.T) { { key: sourceC, gvks: []schema.GroupVersionKind{nsGVK}, - // this call will not error out even though we added a non existent gvk previously to different sync source. + // this call will not error out even though we previously added a non existent gvk to a different sync source. + // this way the errors in watch manager caused by one sync source do not impact the other if they are not related + // to the gvks it specifies. }, }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK, nonExistentGVK, nsGVK}, @@ -617,6 +619,11 @@ func Test_interpretErr(t *testing.T) { gvk1 := schema.GroupVersionKind{Group: "g1", Version: "v1", Kind: "k1"} gvk2 := schema.GroupVersionKind{Group: "g2", Version: "v2", Kind: "k2"} someErr := errors.New("some err") + gvk1Err := watch.NewErrorList() + gvk1Err.AddGVKErr(gvk1, someErr) + genErr := watch.NewErrorList() + genErr.Add(someErr) + genErr.AddGVKErr(gvk1, someErr) cases := []struct { name string @@ -631,15 +638,14 @@ func Test_interpretErr(t *testing.T) { }, { name: "intersection exists", - inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithGVKsErr([]schema.GroupVersionKind{gvk1}, someErr))), + inputErr: fmt.Errorf("some err: %w", gvk1Err), inputGVK: []schema.GroupVersionKind{gvk1}, expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, }, { - name: "intersection does not exist", - inputErr: fakes.WatchesErr(fakes.WithGVKsErr([]schema.GroupVersionKind{gvk1}, someErr)), - inputGVK: []schema.GroupVersionKind{gvk2}, - expectedFailingGVKs: nil, + name: "intersection does not exist", + inputErr: gvk1Err, + inputGVK: []schema.GroupVersionKind{gvk2}, }, { name: "non-watchmanager error reports general error with no GVKs", @@ -649,7 +655,7 @@ func Test_interpretErr(t *testing.T) { }, { name: "general error with failing gvks too", - inputErr: fmt.Errorf("some err: %w", fakes.WatchesErr(fakes.WithErr(someErr), fakes.WithGVKsErr([]schema.GroupVersionKind{gvk1}, someErr))), + inputErr: fmt.Errorf("some err: %w", genErr), inputGVK: []schema.GroupVersionKind{gvk1, gvk2}, expectGeneral: true, }, diff --git a/pkg/fakes/fakecfdataclient.go b/pkg/fakes/fakecfdataclient.go index ba81b4fe255..bdd523eaea0 100644 --- a/pkg/fakes/fakecfdataclient.go +++ b/pkg/fakes/fakecfdataclient.go @@ -20,6 +20,7 @@ import ( constraintTypes "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -144,3 +145,27 @@ func (f *FakeCfClient) SetErroring(enabled bool) { defer f.mu.Unlock() f.needsToError = enabled } + +func UnstructuredFor(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + u.SetName(name) + if namespace == "" { + u.SetNamespace("default") + } else { + u.SetNamespace(namespace) + } + + if gvk.Kind == "Pod" { + u.Object["spec"] = map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "foo-container", + "image": "foo-image", + }, + }, + } + } + + return u +} diff --git a/pkg/fakes/sync.go b/pkg/fakes/sync.go index 293b92486f9..12b5933fa20 100644 --- a/pkg/fakes/sync.go +++ b/pkg/fakes/sync.go @@ -5,7 +5,6 @@ import ( syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -57,27 +56,3 @@ func SyncSetFor(name string, kinds []schema.GroupVersionKind) *syncsetv1alpha1.S }, } } - -func UnstructuredFor(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - u.SetName(name) - if namespace == "" { - u.SetNamespace("default") - } else { - u.SetNamespace(namespace) - } - - if gvk.Kind == "Pod" { - u.Object["spec"] = map[string]interface{}{ - "containers": []map[string]interface{}{ - { - "name": "foo-container", - "image": "foo-image", - }, - }, - } - } - - return u -} diff --git a/pkg/fakes/watcherr.go b/pkg/fakes/watcherr.go deleted file mode 100644 index 50d7a23ec33..00000000000 --- a/pkg/fakes/watcherr.go +++ /dev/null @@ -1,32 +0,0 @@ -package fakes - -import ( - "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -type FOpt func(f *watch.ErrorList) - -func WithErr(e error) FOpt { - return func(f *watch.ErrorList) { - f.Add(e) - } -} - -func WithGVKsErr(gvks []schema.GroupVersionKind, e error) FOpt { - return func(f *watch.ErrorList) { - for _, gvk := range gvks { - f.AddGVKErr(gvk, e) - } - } -} - -func WatchesErr(opts ...FOpt) error { - result := watch.NewErrorList() - - for _, opt := range opts { - opt(result) - } - - return result -} diff --git a/pkg/readiness/pruner/pruner_test.go b/pkg/readiness/pruner/pruner_test.go index c1a5d60f0f0..993d548736f 100644 --- a/pkg/readiness/pruner/pruner_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -50,11 +50,18 @@ func TestMain(m *testing.M) { } type testOptions struct { - startControllers bool - testLister readiness.Lister + addConstrollers bool + addExpectationsPruner bool + testLister readiness.Lister } -func setupTest(ctx context.Context, t *testing.T, o testOptions) (*ExpectationsPruner, client.Client) { +type testResources struct { + expectationsPruner *ExpectationsPruner + manager manager.Manager + k8sClient client.Client +} + +func setupTest(ctx context.Context, t *testing.T, o testOptions) *testResources { t.Helper() mgr, wm := testutils.SetupManager(t, cfg) @@ -93,7 +100,7 @@ func setupTest(ctx context.Context, t *testing.T, o testOptions) (*ExpectationsP cm, err := cachemanager.NewCacheManager(config) require.NoError(t, err, "creating cachemanager") - if !o.startControllers { + if !o.addConstrollers { // need to start the cachemanager if controllers are not started // since the cachemanager is started in the controllers code. require.NoError(t, mgr.Add(cm), "adding cachemanager as a runnable") @@ -117,12 +124,15 @@ func setupTest(ctx context.Context, t *testing.T, o testOptions) (*ExpectationsP require.NoError(t, controller.AddToManager(mgr, &opts), "registering controllers") } - testutils.StartManager(ctx, t, mgr) - - return &ExpectationsPruner{ + ep := &ExpectationsPruner{ cacheMgr: cm, tracker: tracker, - }, c + } + if o.addExpectationsPruner { + require.NoError(t, mgr.Add(ep), "adding expectationspruner as a runnable") + } + + return &testResources{expectationsPruner: ep, manager: mgr, k8sClient: c} } // Test_ExpectationsMgr_DeletedSyncSets tests scenarios in which SyncSet and Config resources @@ -163,10 +173,12 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { require.NoError(t, testutils.ApplyFixtures(tt.fixturesPath, cfg), "applying base fixtures") - em, c := setupTest(ctx, t, testOptions{startControllers: tt.startControllers}) + testRes := setupTest(ctx, t, testOptions{addConstrollers: tt.startControllers}) + + testutils.StartManager(ctx, t, testRes.manager) require.Eventually(t, func() bool { - return em.tracker.Populated() + return testRes.expectationsPruner.tracker.Populated() }, timeout, tick, "waiting on tracker to populate") for _, name := range tt.syncsetsToDelete { @@ -174,7 +186,7 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { u.SetGroupVersionKind(syncsetGVK) u.SetName(name) - require.NoError(t, c.Delete(ctx, u), fmt.Sprintf("deleting syncset %s", name)) + require.NoError(t, testRes.k8sClient.Delete(ctx, u), fmt.Sprintf("deleting syncset %s", name)) } if tt.deleteConfig != "" { u := &unstructured.Unstructured{} @@ -182,13 +194,13 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { u.SetNamespace("gatekeeper-system") u.SetName(tt.deleteConfig) - require.NoError(t, c.Delete(ctx, u), fmt.Sprintf("deleting config %s", tt.deleteConfig)) + require.NoError(t, testRes.k8sClient.Delete(ctx, u), fmt.Sprintf("deleting config %s", tt.deleteConfig)) } - em.pruneNotWatchedGVKs() + testRes.expectationsPruner.pruneNotWatchedGVKs() require.Eventually(t, func() bool { - return em.tracker.Satisfied() + return testRes.expectationsPruner.tracker.Satisfied() }, timeout, tick, "waiting on tracker to get satisfied") cancelFunc() @@ -201,23 +213,33 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { func Test_ExpectationsMgr_missedInformers(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) - // because we will use a separate lister for the tracker from the mgr client - // the contents of the readiness tracker will be superset of the contents of the mgr's client + // Set up one data store for readyTracker: + // we will use a separate lister for the tracker from the mgr client and make + // the contents of the readiness tracker be a superset of the contents of the mgr's client tl := fakes.NewTestLister( fakes.WithSyncSets([]*syncsetv1alpha1.SyncSet{ - fakes.SyncSetFor("syncset-1", []schema.GroupVersionKind{podGVK, configMapGVK}), + fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK, configMapGVK}), }), ) - em, _ := setupTest(ctx, t, testOptions{testLister: tl}) + + testRes := setupTest(ctx, t, testOptions{testLister: tl, addExpectationsPruner: true}) + + // Set up another store for the controllers and watchManager + syncsetA := fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK}) + require.NoError(t, testRes.k8sClient.Create(ctx, syncsetA)) + + testutils.StartManager(ctx, t, testRes.manager) require.Eventually(t, func() bool { - return em.tracker.SyncSourcesSatisfied() + return testRes.expectationsPruner.tracker.SyncSourcesSatisfied() }, timeout, tick, "waiting on sync sources to get satisfied") - em.pruneNotWatchedGVKs() + // As configMapGVK is absent from this syncset-1, the CacheManager will never observe configMapGVK + // being deleted and will never cancel the data expectation for configMapGVK. + require.NoError(t, testRes.k8sClient.Delete(ctx, syncsetA)) require.Eventually(t, func() bool { - return em.tracker.Satisfied() + return testRes.expectationsPruner.tracker.Satisfied() }, timeout, tick, "waiting on tracker to get satisfied") cancelFunc() diff --git a/pkg/readiness/ready_tracker_unit_test.go b/pkg/readiness/ready_tracker_unit_test.go index 90746fc533c..63afefccf7d 100644 --- a/pkg/readiness/ready_tracker_unit_test.go +++ b/pkg/readiness/ready_tracker_unit_test.go @@ -173,9 +173,9 @@ func Test_Tracker_TryCancelData(t *testing.T) { l := fakes.NewTestLister( fakes.WithSyncSets([]*syncsetv1alpha1.SyncSet{&testSyncSet}), ) - for _, tt := range tcs { + for _, tc := range tcs { objDataFn := func() objData { - return objData{retries: tt.retries} + return objData{retries: tc.retries} } rt := newTracker(l, false, false, false, objDataFn) @@ -197,7 +197,7 @@ func Test_Tracker_TryCancelData(t *testing.T) { // observe the sync source for readiness rt.syncsets.Observe(&testSyncSet) - for i := tt.retries; i > 0; i-- { + for i := tc.retries; i > 0; i-- { require.False(t, rt.data.Satisfied(), "data tracker should not be satisfied") require.False(t, rt.Satisfied(), fmt.Sprintf("tracker with %d retries should not be satisfied", i)) rt.TryCancelData(podGVK) diff --git a/pkg/watch/errorlist_test.go b/pkg/watch/errorlist_test.go index 6f8b783cc8d..d17da13a5f6 100644 --- a/pkg/watch/errorlist_test.go +++ b/pkg/watch/errorlist_test.go @@ -56,18 +56,18 @@ func Test_WatchesError(t *testing.T) { }, } - for _, tt := range tcs { - t.Run(tt.name, func(t *testing.T) { + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { er := NewErrorList() - for _, gvkErr := range tt.errsToAdd.gvkErrs { + for _, gvkErr := range tc.errsToAdd.gvkErrs { er.AddGVKErr(gvkErr.gvk, gvkErr.err) } - for _, err := range tt.errsToAdd.errs { + for _, err := range tc.errsToAdd.errs { er.Add(err) } - require.ElementsMatch(t, tt.expectedGVKs, er.FailingGVKs()) - require.Equal(t, tt.generalErr, er.HasGeneralErr()) + require.ElementsMatch(t, tc.expectedGVKs, er.FailingGVKs()) + require.Equal(t, tc.generalErr, er.HasGeneralErr()) }) } } From c8537e2bb2e88bd3761e3347db661a419487fcd8 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Sun, 29 Oct 2023 07:04:57 +0000 Subject: [PATCH 23/42] use fake client builder Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/fakes/fakecfdataclient.go | 4 +- pkg/fakes/scheme.go | 20 ++++++++++ pkg/readiness/pruner/pruner_test.go | 40 +++++++++++++++---- .../syncsets-resources/20-syncset-1.yaml | 2 +- pkg/readiness/ready_tracker.go | 2 +- 5 files changed, 56 insertions(+), 12 deletions(-) create mode 100644 pkg/fakes/scheme.go diff --git a/pkg/fakes/fakecfdataclient.go b/pkg/fakes/fakecfdataclient.go index bdd523eaea0..2ad1958d537 100644 --- a/pkg/fakes/fakecfdataclient.go +++ b/pkg/fakes/fakecfdataclient.go @@ -158,8 +158,8 @@ func UnstructuredFor(gvk schema.GroupVersionKind, namespace, name string) *unstr if gvk.Kind == "Pod" { u.Object["spec"] = map[string]interface{}{ - "containers": []map[string]interface{}{ - { + "containers": []interface{}{ + map[string]interface{}{ "name": "foo-container", "image": "foo-image", }, diff --git a/pkg/fakes/scheme.go b/pkg/fakes/scheme.go new file mode 100644 index 00000000000..6279cf8e694 --- /dev/null +++ b/pkg/fakes/scheme.go @@ -0,0 +1,20 @@ +package fakes + +import ( + "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" + "k8s.io/apimachinery/pkg/runtime" +) + +// test scheme for various needs throughout gatekeeper. +var testScheme *runtime.Scheme + +func init() { + testScheme = runtime.NewScheme() + if err := v1beta1.AddToScheme(testScheme); err != nil { + panic(err) + } +} + +func GetTestScheme() *runtime.Scheme { + return testScheme +} diff --git a/pkg/readiness/pruner/pruner_test.go b/pkg/readiness/pruner/pruner_test.go index 993d548736f..6de0ae65bc0 100644 --- a/pkg/readiness/pruner/pruner_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -216,13 +217,9 @@ func Test_ExpectationsMgr_missedInformers(t *testing.T) { // Set up one data store for readyTracker: // we will use a separate lister for the tracker from the mgr client and make // the contents of the readiness tracker be a superset of the contents of the mgr's client - tl := fakes.NewTestLister( - fakes.WithSyncSets([]*syncsetv1alpha1.SyncSet{ - fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK, configMapGVK}), - }), - ) - - testRes := setupTest(ctx, t, testOptions{testLister: tl, addExpectationsPruner: true}) + // the syncset will look like: + // *fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK, configMapGVK}), + testRes := setupTest(ctx, t, testOptions{testLister: makeTestLister(t), addExpectationsPruner: true, addConstrollers: true}) // Set up another store for the controllers and watchManager syncsetA := fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK}) @@ -234,7 +231,7 @@ func Test_ExpectationsMgr_missedInformers(t *testing.T) { return testRes.expectationsPruner.tracker.SyncSourcesSatisfied() }, timeout, tick, "waiting on sync sources to get satisfied") - // As configMapGVK is absent from this syncset-1, the CacheManager will never observe configMapGVK + // As configMapGVK is absent from this syncset-a, the CacheManager will never observe configMapGVK // being deleted and will never cancel the data expectation for configMapGVK. require.NoError(t, testRes.k8sClient.Delete(ctx, syncsetA)) @@ -244,3 +241,30 @@ func Test_ExpectationsMgr_missedInformers(t *testing.T) { cancelFunc() } + +func makeTestLister(t *testing.T) readiness.Lister { + syncsetList := &syncsetv1alpha1.SyncSetList{} + syncsetList.Items = []syncsetv1alpha1.SyncSet{ + *fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK, configMapGVK}), + } + + podList := &unstructured.UnstructuredList{} + podList.SetGroupVersionKind(schema.GroupVersionKind{ + Version: "v1", + Kind: "PodList", + }) + podList.Items = []unstructured.Unstructured{ + *fakes.UnstructuredFor(podGVK, "", "pod1-name"), + } + + configMapList := &unstructured.UnstructuredList{} + configMapList.SetGroupVersionKind(schema.GroupVersionKind{ + Version: "v1", + Kind: "ConfigMapList", + }) + configMapList.Items = []unstructured.Unstructured{ + *fakes.UnstructuredFor(configMapGVK, "", "cm1-name"), + } + + return fake.NewClientBuilder().WithLists(syncsetList, podList, configMapList).Build() +} diff --git a/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml index 60af509f63c..9c795bb7039 100644 --- a/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml +++ b/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml @@ -6,4 +6,4 @@ spec: gvks: - group: "" version: "v1" - kind: "ConfigMap" + kind: "Namespace" diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index 5667ff133be..8dcc693e276 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -525,7 +525,7 @@ func (t *Tracker) collectInvalidExpectations(ctx context.Context) { for _, gvk := range t.DataGVKs() { // retrieve the expectations for this key es := t.data.Get(gvk) - err = t.collectForObjectTracker(ctx, es, nil, fmt.Sprintf("%s/%s", "Data", gvk)) + err = t.collectForObjectTracker(ctx, es, nil, fmt.Sprintf("%s/ %s", "Data", gvk)) if err != nil { log.Error(err, "while collecting for the Data type", "gvk", gvk) continue From 7cd0440eeb8c24dfbe804f3da15d82439803ba65 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:16:37 +0000 Subject: [PATCH 24/42] use fake client builder in readiness pkg too Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/fakes/lister.go | 100 ----------------------- pkg/readiness/ready_tracker_unit_test.go | 55 ++++++++++--- 2 files changed, 43 insertions(+), 112 deletions(-) delete mode 100644 pkg/fakes/lister.go diff --git a/pkg/fakes/lister.go b/pkg/fakes/lister.go deleted file mode 100644 index 856c0c9e82e..00000000000 --- a/pkg/fakes/lister.go +++ /dev/null @@ -1,100 +0,0 @@ -package fakes - -import ( - "context" - - "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" - "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" - syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var scheme *runtime.Scheme - -func init() { - scheme = runtime.NewScheme() - if err := v1beta1.AddToScheme(scheme); err != nil { - panic(err) - } - if err := syncsetv1alpha1.AddToScheme(scheme); err != nil { - panic(err) - } -} - -// Fake lister to use for readiness testing. -type TestLister struct { - templatesToList []*templates.ConstraintTemplate - syncSetsToList []*syncsetv1alpha1.SyncSet -} - -type LOpt func(tl *TestLister) - -func WithConstraintTemplates(t []*templates.ConstraintTemplate) LOpt { - return func(tl *TestLister) { - tl.templatesToList = t - } -} - -func WithSyncSets(s []*syncsetv1alpha1.SyncSet) LOpt { - return func(tl *TestLister) { - tl.syncSetsToList = s - } -} - -func NewTestLister(opts ...LOpt) *TestLister { - tl := &TestLister{} - for _, o := range opts { - o(tl) - } - - return tl -} - -// List implements readiness.Lister. -func (tl *TestLister) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - // failures will be swallowed by readiness.retryAll - switch list := list.(type) { - case *v1beta1.ConstraintTemplateList: - if len(tl.templatesToList) == 0 { - return nil - } - - items := []v1beta1.ConstraintTemplate{} - for _, t := range tl.templatesToList { - i := v1beta1.ConstraintTemplate{} - if err := scheme.Convert(t, &i, nil); err != nil { - return err - } - items = append(items, i) - } - list.Items = items - - case *syncsetv1alpha1.SyncSetList: - if len(tl.syncSetsToList) == 0 { - return nil - } - - items := []syncsetv1alpha1.SyncSet{} - for _, t := range tl.syncSetsToList { - i := syncsetv1alpha1.SyncSet{} - if err := scheme.Convert(t, &i, nil); err != nil { - return err - } - items = append(items, i) - } - list.Items = items - - case *unstructured.UnstructuredList: - if len(tl.syncSetsToList) == 0 { - return nil - } - - list.Items = []unstructured.Unstructured{{}} // return one element per list for unstructured. - default: - return nil - } - - return nil -} diff --git a/pkg/readiness/ready_tracker_unit_test.go b/pkg/readiness/ready_tracker_unit_test.go index 63afefccf7d..38cff3bd403 100644 --- a/pkg/readiness/ready_tracker_unit_test.go +++ b/pkg/readiness/ready_tracker_unit_test.go @@ -23,12 +23,16 @@ import ( "time" "github.com/onsi/gomega" + "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/stretchr/testify/require" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) var ( @@ -61,14 +65,47 @@ var ( podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} ) +type testClientOptions struct { + listSyncSets bool + listConstraintTemplates bool +} + +func makeTestClient(t *testing.T, o testClientOptions) client.WithWatch { + clb := fake.NewClientBuilder() + + if o.listConstraintTemplates { + ctList := &v1beta1.ConstraintTemplateList{} + i := v1beta1.ConstraintTemplate{} + require.NoError(t, fakes.GetTestScheme().Convert(&testConstraintTemplate, &i, nil), "converting template") + ctList.Items = []v1beta1.ConstraintTemplate{i} + + clb.WithLists(ctList) + } + + if o.listSyncSets { + syncsetList := &syncsetv1alpha1.SyncSetList{} + syncsetList.Items = []syncsetv1alpha1.SyncSet{testSyncSet} + + podList := &unstructured.UnstructuredList{} + podList.SetGroupVersionKind(schema.GroupVersionKind{ + Version: "v1", + Kind: "PodList", + }) + podList.Items = []unstructured.Unstructured{ + *fakes.UnstructuredFor(podGVK, "", "pod1-name"), + } + + clb.WithLists(syncsetList, podList) + } + + return clb.Build() +} + // Verify that TryCancelTemplate functions the same as regular CancelTemplate if readinessRetries is set to 0. func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { g := gomega.NewWithT(t) - l := fakes.NewTestLister( - fakes.WithConstraintTemplates([]*templates.ConstraintTemplate{&testConstraintTemplate}), - ) - rt := newTracker(l, false, false, false, func() objData { + rt := newTracker(makeTestClient(t, testClientOptions{listConstraintTemplates: true}), false, false, false, func() objData { return objData{retries: 0} }) @@ -109,10 +146,7 @@ func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { g := gomega.NewWithT(t) - l := fakes.NewTestLister( - fakes.WithConstraintTemplates([]*templates.ConstraintTemplate{&testConstraintTemplate}), - ) - rt := newTracker(l, false, false, false, func() objData { + rt := newTracker(makeTestClient(t, testClientOptions{listConstraintTemplates: true}), false, false, false, func() objData { return objData{retries: 2} }) @@ -170,14 +204,11 @@ func Test_Tracker_TryCancelData(t *testing.T) { {name: "with retries", retries: 2}, } - l := fakes.NewTestLister( - fakes.WithSyncSets([]*syncsetv1alpha1.SyncSet{&testSyncSet}), - ) for _, tc := range tcs { objDataFn := func() objData { return objData{retries: tc.retries} } - rt := newTracker(l, false, false, false, objDataFn) + rt := newTracker(makeTestClient(t, testClientOptions{listSyncSets: true}), false, false, false, objDataFn) ctx, cancel := context.WithCancel(context.Background()) var runErr error From c4ea7cb44989e8e109f0b88a8b4e189f1e62f45b Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 2 Nov 2023 20:41:57 +0000 Subject: [PATCH 25/42] review feedback - failedGVKs for a sync source on Remove - remove controller switch from syncset controller - return errors in trackSyncSources - use fake client with runtime objects - don't use helper functions for listers - tests naming, comments Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- apis/syncset/v1alpha1/syncset_types.go | 2 +- .../bases/syncset.gatekeeper.sh_syncsets.yaml | 4 +- .../syncset-customresourcedefinition.yaml | 2 +- manifest_staging/deploy/gatekeeper.yaml | 2 +- pkg/cachemanager/aggregator/aggregator.go | 21 +++ .../aggregator/aggregator_test.go | 80 ++++++++- pkg/cachemanager/cachemanager.go | 3 +- pkg/cachemanager/cachemanager_test.go | 84 +++++----- pkg/controller/syncset/syncset_controller.go | 27 +--- .../syncset/syncset_controller_test.go | 70 ++++---- pkg/fakes/fakecfdataclient.go | 45 +++--- pkg/fakes/sync.go | 34 ---- pkg/fakes/unstructured.go | 30 ++++ pkg/readiness/pruner/pruner.go | 6 +- pkg/readiness/pruner/pruner_test.go | 68 +++----- pkg/readiness/ready_tracker.go | 29 ++-- pkg/readiness/ready_tracker_test.go | 7 +- pkg/readiness/ready_tracker_unit_test.go | 152 ++++++++---------- pkg/watch/errorlist_test.go | 6 +- 19 files changed, 339 insertions(+), 333 deletions(-) create mode 100644 pkg/fakes/unstructured.go diff --git a/apis/syncset/v1alpha1/syncset_types.go b/apis/syncset/v1alpha1/syncset_types.go index a542e4691c4..c2079d81214 100644 --- a/apis/syncset/v1alpha1/syncset_types.go +++ b/apis/syncset/v1alpha1/syncset_types.go @@ -26,7 +26,7 @@ func (e *GVKEntry) ToGroupVersionKind() schema.GroupVersionKind { // +kubebuilder:resource:scope=Cluster // +kubebuilder:object:root=true -// SyncSet is the Schema for the SyncSet API. +// SyncSet defines which resources Gatekeeper will cache. The union of all SyncSets plus the syncOnly field of Gatekeeper's Config resource defines the sets of resources that will be synced. type SyncSet struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/config/crd/bases/syncset.gatekeeper.sh_syncsets.yaml b/config/crd/bases/syncset.gatekeeper.sh_syncsets.yaml index 790fb2a7436..08ec5fc832e 100644 --- a/config/crd/bases/syncset.gatekeeper.sh_syncsets.yaml +++ b/config/crd/bases/syncset.gatekeeper.sh_syncsets.yaml @@ -18,7 +18,9 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: - description: SyncSet is the Schema for the SyncSet API. + description: SyncSet defines which resources Gatekeeper will cache. The union + of all SyncSets plus the syncOnly field of Gatekeeper's Config resource + defines the sets of resources that will be synced. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation diff --git a/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml b/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml index d239b742ab6..c5c51f9da4a 100644 --- a/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml +++ b/manifest_staging/charts/gatekeeper/crds/syncset-customresourcedefinition.yaml @@ -19,7 +19,7 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: - description: SyncSet is the Schema for the SyncSet API. + description: SyncSet defines which resources Gatekeeper will cache. The union of all SyncSets plus the syncOnly field of Gatekeeper's Config resource defines the sets of resources that will be synced. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' diff --git a/manifest_staging/deploy/gatekeeper.yaml b/manifest_staging/deploy/gatekeeper.yaml index 7a6b23f510b..3e0be6969ae 100644 --- a/manifest_staging/deploy/gatekeeper.yaml +++ b/manifest_staging/deploy/gatekeeper.yaml @@ -3392,7 +3392,7 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: - description: SyncSet is the Schema for the SyncSet API. + description: SyncSet defines which resources Gatekeeper will cache. The union of all SyncSets plus the syncOnly field of Gatekeeper's Config resource defines the sets of resources that will be synced. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index ee7747fad39..549e57dfc1d 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -116,6 +116,27 @@ func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} { return cpy } +// List returnes the gvks for a given Key that are not referenced by any other Key. +func (b *GVKAgreggator) ListNotShared(k Key) []schema.GroupVersionKind { + b.mu.RLock() + defer b.mu.RUnlock() + + gvks := b.store[k] + gvksToReturn := []schema.GroupVersionKind{} + for gvk := range gvks { + keys, ok := b.reverseStore[gvk] + if !ok { + continue + } + + if len(keys) == 1 { // by definition this Key k is the only one referencing this gvk + gvksToReturn = append(gvksToReturn, gvk) + } + } + + return gvksToReturn +} + // GVKs returns a list of all of the schema.GroupVersionKind that are aggregated. func (b *GVKAgreggator) GVKs() []schema.GroupVersionKind { b.mu.RLock() diff --git a/pkg/cachemanager/aggregator/aggregator_test.go b/pkg/cachemanager/aggregator/aggregator_test.go index 22dec711065..af8bc4fd7d2 100644 --- a/pkg/cachemanager/aggregator/aggregator_test.go +++ b/pkg/cachemanager/aggregator/aggregator_test.go @@ -34,7 +34,7 @@ var ( g3v1k2 = schema.GroupVersionKind{Group: "group3", Version: "v1", Kind: "Kind2"} ) -type upsertKeyGVKs struct { +type keyedGVKs struct { key Key gvks []schema.GroupVersionKind } @@ -43,14 +43,14 @@ func Test_GVKAggregator_Upsert(t *testing.T) { tests := []struct { name string // each entry in the list is a new Upsert call - keyGVKs []upsertKeyGVKs + keyGVKs []keyedGVKs expectData map[Key]map[schema.GroupVersionKind]struct{} expectRev map[schema.GroupVersionKind]map[Key]struct{} }{ { name: "empty GVKs", - keyGVKs: []upsertKeyGVKs{ + keyGVKs: []keyedGVKs{ { key: Key{ Source: syncset, @@ -64,7 +64,7 @@ func Test_GVKAggregator_Upsert(t *testing.T) { }, { name: "add one key and GVKs", - keyGVKs: []upsertKeyGVKs{ + keyGVKs: []keyedGVKs{ { key: Key{ Source: syncset, @@ -99,7 +99,7 @@ func Test_GVKAggregator_Upsert(t *testing.T) { }, { name: "add two keys and GVKs", - keyGVKs: []upsertKeyGVKs{ + keyGVKs: []keyedGVKs{ { key: Key{ Source: syncset, @@ -156,7 +156,7 @@ func Test_GVKAggregator_Upsert(t *testing.T) { }, { name: "add one key and overwrite it", - keyGVKs: []upsertKeyGVKs{ + keyGVKs: []keyedGVKs{ { key: Key{ Source: syncset, @@ -273,6 +273,74 @@ func Test_GVKAgreggator_Remove(t *testing.T) { }) } +func Test_GVKAggregator_ListNotShared(t *testing.T) { + syncSetKey := Key{Source: syncset, ID: "foo"} // we will list this key + configKey := Key{Source: configsync, ID: "foo"} + + tests := []struct { + name string + keyGVKs []keyedGVKs + + expectedNotShared []schema.GroupVersionKind + }{ + { + name: "all gvks shared", + keyGVKs: []keyedGVKs{ + { + key: syncSetKey, + gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}, + }, + { + key: configKey, + gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}, + }, + }, + expectedNotShared: []schema.GroupVersionKind{}, + }, + { + name: "no gvks shared", + keyGVKs: []keyedGVKs{ + { + key: syncSetKey, + gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}, + }, + { + key: configKey, + gvks: []schema.GroupVersionKind{g2v1k1, g2v1k2}, + }, + }, + expectedNotShared: []schema.GroupVersionKind{g1v1k1, g1v1k2}, + }, + { + name: "some gvks shared", + keyGVKs: []keyedGVKs{ + { + key: syncSetKey, + gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}, + }, + { + key: configKey, + gvks: []schema.GroupVersionKind{g1v1k1, g2v1k2}, + }, + }, + expectedNotShared: []schema.GroupVersionKind{g1v1k2}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt := tt + agg := NewGVKAggregator() + + for _, keyGVKs := range tt.keyGVKs { + require.NoError(t, agg.Upsert(keyGVKs.key, keyGVKs.gvks)) + } + + require.ElementsMatch(t, tt.expectedNotShared, agg.ListNotShared(syncSetKey)) + }) + } +} + // Test_GVKAggreggator_E2E is a test that: // - Upserts two sources with different GVKs // - Upserts the two sources with some overlapping GVKs diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 28a73d89120..91337a04430 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -201,12 +201,13 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke c.mu.Lock() defer c.mu.Unlock() + gvksNotShared := c.gvksToSync.ListNotShared(sourceKey) if err := c.gvksToSync.Remove(sourceKey); err != nil { return fmt.Errorf("internal error removing source: %w", err) } err := c.replaceWatchSet(ctx) - if general, _ := interpretErr(err, []schema.GroupVersionKind{}); general { + if general, failedGVKs := interpretErr(err, gvksNotShared); general || len(failedGVKs) > 0 { return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index cae59edb008..d000f702f3d 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -33,9 +33,9 @@ var ( nsGVK = schema.GroupVersionKind{Version: "v1", Kind: "Namespace"} nonExistentGVK = schema.GroupVersionKind{Version: "v1", Kind: "DoesNotExist"} - sourceA = aggregator.Key{Source: "a", ID: "source"} - sourceB = aggregator.Key{Source: "b", ID: "source"} - sourceC = aggregator.Key{Source: "c", ID: "source"} + configKey = aggregator.Key{Source: "config", ID: "config"} + syncsetAKey = aggregator.Key{Source: "syncset", ID: "a"} + syncsetBkey = aggregator.Key{Source: "syncset", ID: "b"} ) func TestMain(m *testing.M) { @@ -380,7 +380,7 @@ func TestCacheManager_UpsertSource(t *testing.T) { name: "add one source", sources: []source{ { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{configMapGVK}, }, }, @@ -390,11 +390,11 @@ func TestCacheManager_UpsertSource(t *testing.T) { name: "overwrite source", sources: []source{ { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{configMapGVK}, }, { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{podGVK}, }, }, @@ -404,11 +404,11 @@ func TestCacheManager_UpsertSource(t *testing.T) { name: "remove GVK from a source", sources: []source{ { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{configMapGVK}, }, { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{}, }, }, @@ -418,11 +418,11 @@ func TestCacheManager_UpsertSource(t *testing.T) { name: "add two disjoint sources", sources: []source{ { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{configMapGVK}, }, { - key: sourceB, + key: syncsetAKey, gvks: []schema.GroupVersionKind{podGVK}, }, }, @@ -432,11 +432,11 @@ func TestCacheManager_UpsertSource(t *testing.T) { name: "add two sources with fully overlapping gvks", sources: []source{ { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{podGVK}, }, { - key: sourceB, + key: syncsetAKey, gvks: []schema.GroupVersionKind{podGVK}, }, }, @@ -446,11 +446,11 @@ func TestCacheManager_UpsertSource(t *testing.T) { name: "add two sources with partially overlapping gvks", sources: []source{ { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{configMapGVK, podGVK}, }, { - key: sourceB, + key: syncsetAKey, gvks: []schema.GroupVersionKind{podGVK}, }, }, @@ -474,9 +474,9 @@ func TestCacheManager_UpsertSource(t *testing.T) { func TestCacheManager_UpsertSource_errorcases(t *testing.T) { type source struct { - key aggregator.Key - gvks []schema.GroupVersionKind - err error + key aggregator.Key + gvks []schema.GroupVersionKind + wantErr bool } tcs := []struct { @@ -488,17 +488,17 @@ func TestCacheManager_UpsertSource_errorcases(t *testing.T) { name: "add two sources where one fails to establish all watches", sources: []source{ { - key: sourceA, + key: configKey, gvks: []schema.GroupVersionKind{configMapGVK}, }, { - key: sourceB, + key: syncsetAKey, gvks: []schema.GroupVersionKind{podGVK, nonExistentGVK}, // UpsertSource will err out because of nonExistentGVK - err: errors.New("error for gvk: /v1, Kind=DoesNotExist: adding watch for /v1, Kind=DoesNotExist getting informer for kind: /v1, Kind=DoesNotExist no matches for kind \"DoesNotExist\" in version \"v1\""), + wantErr: true, }, { - key: sourceC, + key: syncsetBkey, gvks: []schema.GroupVersionKind{nsGVK}, // this call will not error out even though we previously added a non existent gvk to a different sync source. // this way the errors in watch manager caused by one sync source do not impact the other if they are not related @@ -514,8 +514,8 @@ func TestCacheManager_UpsertSource_errorcases(t *testing.T) { cacheManager, ctx := makeCacheManager(t) for _, source := range tc.sources { - if source.err != nil { - require.ErrorContains(t, cacheManager.UpsertSource(ctx, source.key, source.gvks), source.err.Error(), fmt.Sprintf("while upserting source: %s", source.key)) + if source.wantErr { + require.Error(t, cacheManager.UpsertSource(ctx, source.key, source.gvks), fmt.Sprintf("while upserting source: %s", source.key)) } else { require.NoError(t, cacheManager.UpsertSource(ctx, source.key, source.gvks), fmt.Sprintf("while upserting source: %s", source.key)) } @@ -538,62 +538,62 @@ func TestCacheManager_RemoveSource(t *testing.T) { { name: "remove disjoint source", existingSources: []source{ - {sourceA, []schema.GroupVersionKind{podGVK}}, - {sourceB, []schema.GroupVersionKind{configMapGVK}}, + {configKey, []schema.GroupVersionKind{podGVK}}, + {syncsetAKey, []schema.GroupVersionKind{configMapGVK}}, }, - sourcesToRemove: []aggregator.Key{sourceB}, + sourcesToRemove: []aggregator.Key{syncsetAKey}, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { name: "remove fully overlapping source", existingSources: []source{ - {sourceA, []schema.GroupVersionKind{podGVK}}, - {sourceB, []schema.GroupVersionKind{podGVK}}, + {configKey, []schema.GroupVersionKind{podGVK}}, + {syncsetAKey, []schema.GroupVersionKind{podGVK}}, }, - sourcesToRemove: []aggregator.Key{sourceB}, + sourcesToRemove: []aggregator.Key{syncsetAKey}, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { name: "remove partially overlapping source", existingSources: []source{ - {sourceA, []schema.GroupVersionKind{podGVK}}, - {sourceB, []schema.GroupVersionKind{podGVK, configMapGVK}}, + {configKey, []schema.GroupVersionKind{podGVK}}, + {syncsetAKey, []schema.GroupVersionKind{podGVK, configMapGVK}}, }, - sourcesToRemove: []aggregator.Key{sourceA}, + sourcesToRemove: []aggregator.Key{configKey}, expectedGVKs: []schema.GroupVersionKind{podGVK, configMapGVK}, }, { name: "remove non existing source", existingSources: []source{ - {sourceA, []schema.GroupVersionKind{podGVK}}, + {configKey, []schema.GroupVersionKind{podGVK}}, }, - sourcesToRemove: []aggregator.Key{sourceB}, + sourcesToRemove: []aggregator.Key{syncsetAKey}, expectedGVKs: []schema.GroupVersionKind{podGVK}, }, { name: "remove source with a non existing gvk", existingSources: []source{ - {sourceA, []schema.GroupVersionKind{nonExistentGVK}}, + {configKey, []schema.GroupVersionKind{nonExistentGVK}}, }, - sourcesToRemove: []aggregator.Key{sourceA}, + sourcesToRemove: []aggregator.Key{configKey}, expectedGVKs: []schema.GroupVersionKind{}, }, { name: "remove source from a watch set with a non existing gvk", existingSources: []source{ - {sourceA, []schema.GroupVersionKind{nonExistentGVK}}, - {sourceB, []schema.GroupVersionKind{podGVK}}, + {configKey, []schema.GroupVersionKind{nonExistentGVK}}, + {syncsetAKey, []schema.GroupVersionKind{podGVK}}, }, - sourcesToRemove: []aggregator.Key{sourceB}, + sourcesToRemove: []aggregator.Key{syncsetAKey}, expectedGVKs: []schema.GroupVersionKind{nonExistentGVK}, }, { name: "remove source with non existent gvk from a watch set with a remaining non existing gvk", existingSources: []source{ - {sourceA, []schema.GroupVersionKind{nonExistentGVK}}, - {sourceB, []schema.GroupVersionKind{nonExistentGVK}}, + {configKey, []schema.GroupVersionKind{nonExistentGVK}}, + {syncsetAKey, []schema.GroupVersionKind{nonExistentGVK}}, }, - sourcesToRemove: []aggregator.Key{sourceB}, + sourcesToRemove: []aggregator.Key{syncsetAKey}, expectedGVKs: []schema.GroupVersionKind{nonExistentGVK}, }, } diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go index c96eb3753d4..5847c8cd180 100644 --- a/pkg/controller/syncset/syncset_controller.go +++ b/pkg/controller/syncset/syncset_controller.go @@ -34,19 +34,17 @@ var ( ) type Adder struct { - CacheManager *cm.CacheManager - ControllerSwitch *watch.ControllerSwitch - Tracker *readiness.Tracker + CacheManager *cm.CacheManager + Tracker *readiness.Tracker } -// Add creates a new SyncSetController and adds it to the Manager with default RBAC. The Manager will set fields on the Controller -// and Start it when the Manager is Started. +// Add creates a new controller for SyncSets and adds it to the Manager. func (a *Adder) Add(mgr manager.Manager) error { if !operations.HasValidationOperations() { return nil } - r, err := newReconciler(mgr, a.CacheManager, a.ControllerSwitch, a.Tracker) + r, err := newReconciler(mgr, a.CacheManager, a.Tracker) if err != nil { return err } @@ -59,14 +57,13 @@ func (a *Adder) InjectCacheManager(o *cm.CacheManager) { } func (a *Adder) InjectControllerSwitch(cs *watch.ControllerSwitch) { - a.ControllerSwitch = cs } func (a *Adder) InjectTracker(t *readiness.Tracker) { a.Tracker = t } -func newReconciler(mgr manager.Manager, cm *cm.CacheManager, cs *watch.ControllerSwitch, tracker *readiness.Tracker) (*ReconcileSyncSet, error) { +func newReconciler(mgr manager.Manager, cm *cm.CacheManager, tracker *readiness.Tracker) (*ReconcileSyncSet, error) { if cm == nil { return nil, fmt.Errorf("CacheManager must be non-nil") } @@ -77,7 +74,6 @@ func newReconciler(mgr manager.Manager, cm *cm.CacheManager, cs *watch.Controlle return &ReconcileSyncSet{ reader: mgr.GetClient(), scheme: mgr.GetScheme(), - cs: cs, cacheManager: cm, tracker: tracker, }, nil @@ -105,19 +101,10 @@ type ReconcileSyncSet struct { scheme *runtime.Scheme cacheManager *cm.CacheManager - cs *watch.ControllerSwitch tracker *readiness.Tracker } func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - // Short-circuit if shutting down. - if r.cs != nil { - defer r.cs.Exit() - if !r.cs.Enter() { - return reconcile.Result{}, nil - } - } - syncsetTr := r.tracker.For(syncsetGVK) exists := true syncset := &syncsetv1alpha1.SyncSet{} @@ -137,7 +124,7 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ if err := r.cacheManager.RemoveSource(ctx, sk); err != nil { syncsetTr.TryCancelExpect(syncset) - return reconcile.Result{}, fmt.Errorf("synceset-controller: error removing source: %w", err) + return reconcile.Result{}, fmt.Errorf("syncset-controller: error removing source: %w", err) } syncsetTr.CancelExpect(syncset) @@ -152,7 +139,7 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ if err := r.cacheManager.UpsertSource(ctx, sk, gvks); err != nil { syncsetTr.TryCancelExpect(syncset) - return reconcile.Result{Requeue: true}, fmt.Errorf("synceset-controller: error upserting watches: %w", err) + return reconcile.Result{Requeue: true}, fmt.Errorf("syncset-controller: error upserting watches: %w", err) } syncsetTr.Observe(syncset) diff --git a/pkg/controller/syncset/syncset_controller_test.go b/pkg/controller/syncset/syncset_controller_test.go index 375d1030052..cd39a00c72c 100644 --- a/pkg/controller/syncset/syncset_controller_test.go +++ b/pkg/controller/syncset/syncset_controller_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" + syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" - "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" @@ -35,9 +35,9 @@ const ( tick = time.Second * 2 ) -// Test_ReconcileSyncSet_wConfigController verifies that SyncSet and Config resources +// Test_ReconcileSyncSet verifies that SyncSet resources // can get reconciled and their respective specs are added to the data client. -func Test_ReconcileSyncSet_wConfigController(t *testing.T) { +func Test_ReconcileSyncSet(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() @@ -52,13 +52,6 @@ func Test_ReconcileSyncSet_wConfigController(t *testing.T) { syncAdder := syncc.Adder{CacheManager: testRes.cacheMgr, Events: testRes.events} require.NoError(t, syncAdder.Add(*testRes.mgr), "adding sync reconciler to mgr") - configAdder := config.Adder{ - CacheManager: testRes.cacheMgr, - ControllerSwitch: testRes.cs, - Tracker: testRes.tracker, - } - require.NoError(t, configAdder.Add(*testRes.mgr), "adding config reconciler to mgr") - testutils.StartManager(ctx, t, *testRes.mgr) require.NoError(t, testRes.k8sclient.Create(ctx, configMap), fmt.Sprintf("creating ConfigMap %s", "cm1-mame")) @@ -66,52 +59,60 @@ func Test_ReconcileSyncSet_wConfigController(t *testing.T) { tts := []struct { name string - syncSources []client.Object + syncSources []*syncsetv1alpha1.SyncSet expectedGVKs []schema.GroupVersionKind }{ { - name: "config and 1 sync", - syncSources: []client.Object{ - fakes.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}), - fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{podGVK}), - }, - expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK, nsGVK}, - }, - { - name: "config only", - syncSources: []client.Object{ - fakes.ConfigFor([]schema.GroupVersionKind{configMapGVK, nsGVK}), + name: "basic reconcile", + syncSources: []*syncsetv1alpha1.SyncSet{ + fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{configMapGVK, nsGVK}), }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, nsGVK}, }, { - name: "syncset only", - syncSources: []client.Object{ + name: "syncset adjusts spec", + syncSources: []*syncsetv1alpha1.SyncSet{ fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{configMapGVK, nsGVK}), + fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{nsGVK}), }, - expectedGVKs: []schema.GroupVersionKind{configMapGVK, nsGVK}, + expectedGVKs: []schema.GroupVersionKind{nsGVK}, }, } for _, tt := range tts { t.Run(tt.name, func(t *testing.T) { + created := map[string]struct{}{} for _, o := range tt.syncSources { - require.NoError(t, testRes.k8sclient.Create(ctx, o)) + if _, ok := created[o.GetName()]; ok { + curObj, ok := o.DeepCopyObject().(client.Object) + require.True(t, ok) + // eventually we should find the object + require.Eventually(t, func() bool { + return testRes.k8sclient.Get(ctx, client.ObjectKeyFromObject(curObj), curObj) == nil + }, timeout, tick, fmt.Sprintf("getting %s", o.GetName())) + + o.SetResourceVersion(curObj.GetResourceVersion()) + require.NoError(t, testRes.k8sclient.Update(ctx, o), fmt.Sprintf("updating %s", o.GetName())) + } else { + require.NoError(t, testRes.k8sclient.Create(ctx, o), fmt.Sprintf("creating %s", o.GetName())) + created[o.GetName()] = struct{}{} + } } assert.Eventually(t, func() bool { - for _, gvk := range tt.expectedGVKs { - if !testRes.cfClient.HasGVK(gvk) { - return false - } - } - return true + return testRes.cfClient.ContainsGVKs(tt.expectedGVKs) }, timeout, tick) // empty the cache to not leak state between tests + deleted := map[string]struct{}{} for _, o := range tt.syncSources { + if _, ok := deleted[o.GetName()]; ok { + continue + } require.NoError(t, testRes.k8sclient.Delete(ctx, o)) + deleted[o.GetName()] = struct{}{} } + require.Eventually(t, func() bool { return testRes.cfClient.Len() == 0 }, timeout, tick, "could not cleanup") @@ -126,7 +127,6 @@ type testResources struct { wm *watch.Manager cfClient *fakes.FakeCfClient events chan event.GenericEvent - cs *watch.ControllerSwitch tracker *readiness.Tracker } @@ -138,8 +138,6 @@ func setupTest(ctx context.Context, t *testing.T) testResources { testRes := testResources{} - cs := watch.NewSwitch() - testRes.cs = cs tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) testRes.tracker = tracker @@ -166,7 +164,7 @@ func setupTest(ctx context.Context, t *testing.T) testResources { testRes.k8sclient = c testRes.wm = wm - rec, err := newReconciler(mgr, cm, cs, tracker) + rec, err := newReconciler(mgr, cm, tracker) require.NoError(t, err) require.NoError(t, add(mgr, rec)) diff --git a/pkg/fakes/fakecfdataclient.go b/pkg/fakes/fakecfdataclient.go index 2ad1958d537..df16c4c9e71 100644 --- a/pkg/fakes/fakecfdataclient.go +++ b/pkg/fakes/fakecfdataclient.go @@ -20,7 +20,6 @@ import ( constraintTypes "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -132,6 +131,26 @@ func (f *FakeCfClient) HasGVK(gvk schema.GroupVersionKind) bool { return false } +// ContainsGVKs returns true if the cache has data for the gvks given and those gvks only. +func (f *FakeCfClient) ContainsGVKs(gvks []schema.GroupVersionKind) bool { + f.mu.Lock() + defer f.mu.Unlock() + + gvkMap := map[schema.GroupVersionKind]struct{}{} + for _, gvk := range gvks { + gvkMap[gvk] = struct{}{} + } + + foundMap := map[schema.GroupVersionKind]struct{}{} + for k := range f.data { + if _, found := gvkMap[k.Gvk]; !found { + return false + } + foundMap[k.Gvk] = struct{}{} + } + return len(foundMap) == len(gvkMap) +} + // Len returns the number of items in the cache. func (f *FakeCfClient) Len() int { f.mu.Lock() @@ -145,27 +164,3 @@ func (f *FakeCfClient) SetErroring(enabled bool) { defer f.mu.Unlock() f.needsToError = enabled } - -func UnstructuredFor(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - u.SetName(name) - if namespace == "" { - u.SetNamespace("default") - } else { - u.SetNamespace(namespace) - } - - if gvk.Kind == "Pod" { - u.Object["spec"] = map[string]interface{}{ - "containers": []interface{}{ - map[string]interface{}{ - "name": "foo-container", - "image": "foo-image", - }, - }, - } - } - - return u -} diff --git a/pkg/fakes/sync.go b/pkg/fakes/sync.go index 12b5933fa20..882ae52cee6 100644 --- a/pkg/fakes/sync.go +++ b/pkg/fakes/sync.go @@ -1,45 +1,11 @@ package fakes import ( - configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" - "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) -// ConfigFor returns a config resource that watches the requested set of resources. -func ConfigFor(kinds []schema.GroupVersionKind) *configv1alpha1.Config { - entries := make([]configv1alpha1.SyncOnlyEntry, len(kinds)) - for i := range kinds { - entries[i].Group = kinds[i].Group - entries[i].Version = kinds[i].Version - entries[i].Kind = kinds[i].Kind - } - - return &configv1alpha1.Config{ - TypeMeta: metav1.TypeMeta{ - APIVersion: configv1alpha1.GroupVersion.String(), - Kind: "Config", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "config", - Namespace: "gatekeeper-system", - }, - Spec: configv1alpha1.ConfigSpec{ - Sync: configv1alpha1.Sync{ - SyncOnly: entries, - }, - Match: []configv1alpha1.MatchEntry{ - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }, - }, - } -} - // SyncSetFor returns a syncset resource with the given name for the requested set of resources. func SyncSetFor(name string, kinds []schema.GroupVersionKind) *syncsetv1alpha1.SyncSet { entries := make([]syncsetv1alpha1.GVKEntry, len(kinds)) diff --git a/pkg/fakes/unstructured.go b/pkg/fakes/unstructured.go new file mode 100644 index 00000000000..0876da30e4b --- /dev/null +++ b/pkg/fakes/unstructured.go @@ -0,0 +1,30 @@ +package fakes + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func UnstructuredFor(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + u.SetName(name) + if namespace == "" { + u.SetNamespace("default") + } else { + u.SetNamespace(namespace) + } + + if gvk.Kind == "Pod" { + u.Object["spec"] = map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "foo-container", + "image": "foo-image", + }, + }, + } + } + + return u +} diff --git a/pkg/readiness/pruner/pruner.go b/pkg/readiness/pruner/pruner.go index 1c26f2c4c3f..8d864727ed8 100644 --- a/pkg/readiness/pruner/pruner.go +++ b/pkg/readiness/pruner/pruner.go @@ -38,15 +38,15 @@ func (e *ExpectationsPruner) Start(ctx context.Context) error { return nil } if e.tracker.SyncSourcesSatisfied() { - e.pruneNotWatchedGVKs() + e.pruneUnwatchedGVKs() } } } } -// pruneNotWatchedGVKs prunes data expectations that are no longer correct based on the up-to-date +// pruneUnwatchedGVKs prunes data expectations that are no longer correct based on the up-to-date // information in the CacheManager. -func (e *ExpectationsPruner) pruneNotWatchedGVKs() { +func (e *ExpectationsPruner) pruneUnwatchedGVKs() { watchedGVKs := watch.NewSet() watchedGVKs.Add(e.cacheMgr.WatchedGVKs()...) expectedGVKs := watch.NewSet() diff --git a/pkg/readiness/pruner/pruner_test.go b/pkg/readiness/pruner/pruner_test.go index 6de0ae65bc0..31e3af008c5 100644 --- a/pkg/readiness/pruner/pruner_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -51,9 +51,9 @@ func TestMain(m *testing.M) { } type testOptions struct { - addConstrollers bool + addControllers bool addExpectationsPruner bool - testLister readiness.Lister + readyTrackerClient readiness.Lister } type testResources struct { @@ -70,15 +70,14 @@ func setupTest(ctx context.Context, t *testing.T, o testOptions) *testResources var tracker *readiness.Tracker var err error - if o.testLister != nil { - tracker = readiness.NewTracker(o.testLister, false, false, false) - require.NoError(t, mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - return tracker.Run(ctx) - })), "adding tracker to manager") + if o.readyTrackerClient != nil { + tracker = readiness.NewTracker(o.readyTrackerClient, false, false, false) } else { - tracker, err = readiness.SetupTrackerNoReadyz(mgr, false, false, false) - require.NoError(t, err, "setting up tracker") + tracker = readiness.NewTracker(mgr.GetAPIReader(), false, false, false) } + require.NoError(t, mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + return tracker.Run(ctx) + })), "adding tracker to manager") events := make(chan event.GenericEvent, 1024) reg, err := wm.NewRegistrar( @@ -101,7 +100,7 @@ func setupTest(ctx context.Context, t *testing.T, o testOptions) *testResources cm, err := cachemanager.NewCacheManager(config) require.NoError(t, err, "creating cachemanager") - if !o.addConstrollers { + if !o.addControllers { // need to start the cachemanager if controllers are not started // since the cachemanager is started in the controllers code. require.NoError(t, mgr.Add(cm), "adding cachemanager as a runnable") @@ -146,8 +145,6 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { fixturesPath string syncsetsToDelete []string deleteConfig string - // not starting controllers approximates missing events in the informers cache - startControllers bool }{ { name: "delete all syncsets", @@ -160,12 +157,6 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { syncsetsToDelete: []string{"syncset-1"}, deleteConfig: "config", }, - { - name: "delete one syncset", - fixturesPath: "testdata/syncsets-resources", - syncsetsToDelete: []string{"syncset-2"}, - startControllers: true, - }, } for _, tt := range tts { @@ -174,7 +165,7 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { require.NoError(t, testutils.ApplyFixtures(tt.fixturesPath, cfg), "applying base fixtures") - testRes := setupTest(ctx, t, testOptions{addConstrollers: tt.startControllers}) + testRes := setupTest(ctx, t, testOptions{}) testutils.StartManager(ctx, t, testRes.manager) @@ -198,7 +189,7 @@ func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { require.NoError(t, testRes.k8sClient.Delete(ctx, u), fmt.Sprintf("deleting config %s", tt.deleteConfig)) } - testRes.expectationsPruner.pruneNotWatchedGVKs() + testRes.expectationsPruner.pruneUnwatchedGVKs() require.Eventually(t, func() bool { return testRes.expectationsPruner.tracker.Satisfied() @@ -217,9 +208,13 @@ func Test_ExpectationsMgr_missedInformers(t *testing.T) { // Set up one data store for readyTracker: // we will use a separate lister for the tracker from the mgr client and make // the contents of the readiness tracker be a superset of the contents of the mgr's client - // the syncset will look like: - // *fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK, configMapGVK}), - testRes := setupTest(ctx, t, testOptions{testLister: makeTestLister(t), addExpectationsPruner: true, addConstrollers: true}) + lister := fake.NewClientBuilder().WithRuntimeObjects( + fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK, configMapGVK}), + fakes.UnstructuredFor(podGVK, "", "pod1-name"), + fakes.UnstructuredFor(configMapGVK, "", "cm1-name"), + ).Build() + + testRes := setupTest(ctx, t, testOptions{readyTrackerClient: lister, addExpectationsPruner: true, addControllers: true}) // Set up another store for the controllers and watchManager syncsetA := fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK}) @@ -241,30 +236,3 @@ func Test_ExpectationsMgr_missedInformers(t *testing.T) { cancelFunc() } - -func makeTestLister(t *testing.T) readiness.Lister { - syncsetList := &syncsetv1alpha1.SyncSetList{} - syncsetList.Items = []syncsetv1alpha1.SyncSet{ - *fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK, configMapGVK}), - } - - podList := &unstructured.UnstructuredList{} - podList.SetGroupVersionKind(schema.GroupVersionKind{ - Version: "v1", - Kind: "PodList", - }) - podList.Items = []unstructured.Unstructured{ - *fakes.UnstructuredFor(podGVK, "", "pod1-name"), - } - - configMapList := &unstructured.UnstructuredList{} - configMapList.SetGroupVersionKind(schema.GroupVersionKind{ - Version: "v1", - Kind: "ConfigMapList", - }) - configMapList.Items = []unstructured.Unstructured{ - *fakes.UnstructuredFor(configMapGVK, "", "cm1-name"), - } - - return fake.NewClientBuilder().WithLists(syncsetList, podList, configMapList).Build() -} diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index 8dcc693e276..2528559abb4 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -748,7 +748,7 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { cfg, err := t.getConfigResource(ctx) if err != nil { - log.Error(err, "fetching config resource") + return fmt.Errorf("fetching config resource: %w", err) } if cfg == nil { log.Info("config resource not found - skipping for readiness") @@ -773,24 +773,23 @@ func (t *Tracker) trackSyncSources(ctx context.Context) error { syncsets := &syncsetv1alpha1.SyncSetList{} lister := retryLister(t.lister, retryAll) if err := lister.List(ctx, syncsets); err != nil { - log.Error(err, "listing syncsets") - } else { - log.V(logging.DebugLevel).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) + return fmt.Errorf("fetching syncset resources: %w", err) + } - for i := range syncsets.Items { - syncset := syncsets.Items[i] + log.V(logging.DebugLevel).Info("setting expectations for syncsets", "syncsetCount", len(syncsets.Items)) + for i := range syncsets.Items { + syncset := syncsets.Items[i] - t.syncsets.Expect(&syncset) - log.V(logging.DebugLevel).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) + t.syncsets.Expect(&syncset) + log.V(logging.DebugLevel).Info("expecting syncset", "name", syncset.GetName(), "namespace", syncset.GetNamespace()) - for i := range syncset.Spec.GVKs { - gvk := syncset.Spec.GVKs[i].ToGroupVersionKind() - if _, ok := dataGVKs[gvk]; ok { - log.Info("duplicate GVK to sync", "gvk", gvk) - } - - dataGVKs[gvk] = struct{}{} + for i := range syncset.Spec.GVKs { + gvk := syncset.Spec.GVKs[i].ToGroupVersionKind() + if _, ok := dataGVKs[gvk]; ok { + log.Info("duplicate GVK to sync", "gvk", gvk) } + + dataGVKs[gvk] = struct{}{} } } diff --git a/pkg/readiness/ready_tracker_test.go b/pkg/readiness/ready_tracker_test.go index 0da8963cd90..4d017fea68b 100644 --- a/pkg/readiness/ready_tracker_test.go +++ b/pkg/readiness/ready_tracker_test.go @@ -553,7 +553,7 @@ func Test_Tracker(t *testing.T) { // Verifies additional scenarios to the base "testdata/" fixtures, such as // invalid Config resources or overlapping SyncSets, which we want to // make sure the ReadyTracker can handle gracefully. -func Test_Tracker_EdgeCases(t *testing.T) { +func Test_Tracker_SyncSourceEdgeCases(t *testing.T) { tts := []struct { name string fixturesPath string @@ -572,11 +572,6 @@ func Test_Tracker_EdgeCases(t *testing.T) { name: "repeating gvk", fixturesPath: "testdata/config/repeating-gvk", }, - { - // empty syncOnly still needs reconciling for a Config resources - name: "empty sync only gvk", - fixturesPath: "testdata/config/empty-sync", - }, } for _, tt := range tts { diff --git a/pkg/readiness/ready_tracker_unit_test.go b/pkg/readiness/ready_tracker_unit_test.go index 38cff3bd403..39bfa9ca3a1 100644 --- a/pkg/readiness/ready_tracker_unit_test.go +++ b/pkg/readiness/ready_tracker_unit_test.go @@ -22,19 +22,20 @@ import ( "testing" "time" - "github.com/onsi/gomega" "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1beta1" "github.com/open-policy-agent/frameworks/constraint/pkg/core/templates" syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/stretchr/testify/require" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +const ( + timeout = 10 * time.Second + tick = 1 * time.Second +) + var ( testConstraintTemplate = templates.ConstraintTemplate{ ObjectMeta: v1.ObjectMeta{ @@ -62,50 +63,21 @@ var ( }, } - podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + podGVK = testSyncSet.Spec.GVKs[0].ToGroupVersionKind() ) -type testClientOptions struct { - listSyncSets bool - listConstraintTemplates bool -} - -func makeTestClient(t *testing.T, o testClientOptions) client.WithWatch { - clb := fake.NewClientBuilder() - - if o.listConstraintTemplates { - ctList := &v1beta1.ConstraintTemplateList{} - i := v1beta1.ConstraintTemplate{} - require.NoError(t, fakes.GetTestScheme().Convert(&testConstraintTemplate, &i, nil), "converting template") - ctList.Items = []v1beta1.ConstraintTemplate{i} - - clb.WithLists(ctList) - } - - if o.listSyncSets { - syncsetList := &syncsetv1alpha1.SyncSetList{} - syncsetList.Items = []syncsetv1alpha1.SyncSet{testSyncSet} - - podList := &unstructured.UnstructuredList{} - podList.SetGroupVersionKind(schema.GroupVersionKind{ - Version: "v1", - Kind: "PodList", - }) - podList.Items = []unstructured.Unstructured{ - *fakes.UnstructuredFor(podGVK, "", "pod1-name"), - } +var convertedTemplate v1beta1.ConstraintTemplate - clb.WithLists(syncsetList, podList) +func init() { + if err := fakes.GetTestScheme().Convert(&testConstraintTemplate, &convertedTemplate, nil); err != nil { + panic(err) } - - return clb.Build() } // Verify that TryCancelTemplate functions the same as regular CancelTemplate if readinessRetries is set to 0. func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { - g := gomega.NewWithT(t) - - rt := newTracker(makeTestClient(t, testClientOptions{listConstraintTemplates: true}), false, false, false, func() objData { + lister := fake.NewClientBuilder().WithRuntimeObjects(&convertedTemplate).Build() + rt := newTracker(lister, false, false, false, func() objData { return objData{retries: 0} }) @@ -127,9 +99,9 @@ func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { } }) - g.Eventually(func() bool { + require.Eventually(t, func() bool { return rt.Populated() - }, "10s").Should(gomega.BeTrue()) + }, timeout, tick, "waiting for RT to populated") if rt.Satisfied() { t.Fatal("tracker with 0 retries should not be satisfied") @@ -144,9 +116,8 @@ func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { // Verify that TryCancelTemplate must be called enough times to remove all retries before canceling a template. func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { - g := gomega.NewWithT(t) - - rt := newTracker(makeTestClient(t, testClientOptions{listConstraintTemplates: true}), false, false, false, func() objData { + lister := fake.NewClientBuilder().WithRuntimeObjects(&convertedTemplate).Build() + rt := newTracker(lister, false, false, false, func() objData { return objData{retries: 2} }) @@ -168,9 +139,9 @@ func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { } }) - g.Eventually(func() bool { + require.Eventually(t, func() bool { return rt.Populated() - }, "10s").Should(gomega.BeTrue()) + }, timeout, tick, "waiting for RT to populated") if rt.Satisfied() { t.Fatal("tracker with 2 retries should not be satisfied") @@ -196,6 +167,9 @@ func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { } func Test_Tracker_TryCancelData(t *testing.T) { + lister := fake.NewClientBuilder().WithRuntimeObjects( + &testSyncSet, fakes.UnstructuredFor(podGVK, "", "pod1-name"), + ).Build() tcs := []struct { name string retries int @@ -205,46 +179,48 @@ func Test_Tracker_TryCancelData(t *testing.T) { } for _, tc := range tcs { - objDataFn := func() objData { - return objData{retries: tc.retries} - } - rt := newTracker(makeTestClient(t, testClientOptions{listSyncSets: true}), false, false, false, objDataFn) - - ctx, cancel := context.WithCancel(context.Background()) - var runErr error - runWg := sync.WaitGroup{} - runWg.Add(1) - go func() { - runErr = rt.Run(ctx) - // wait for the ready tracker to stop so we don't leak state between tests. - runWg.Done() - }() - - require.Eventually(t, func() bool { - return rt.Populated() - }, 10*time.Second, 1*time.Second, "waiting for RT to populated") - require.False(t, rt.Satisfied(), "tracker with retries should not be satisfied") - - // observe the sync source for readiness - rt.syncsets.Observe(&testSyncSet) - - for i := tc.retries; i > 0; i-- { - require.False(t, rt.data.Satisfied(), "data tracker should not be satisfied") - require.False(t, rt.Satisfied(), fmt.Sprintf("tracker with %d retries should not be satisfied", i)) - rt.TryCancelData(podGVK) - } - require.False(t, rt.Satisfied(), "tracker should not be satisfied") - - rt.TryCancelData(podGVK) // at this point there should no retries - require.True(t, rt.Satisfied(), "tracker with 0 retries and cancellation should be satisfied") - require.True(t, rt.data.Satisfied(), "data tracker should be satisfied") - - _, removed := rt.data.removed[podGVK] - require.True(t, removed, "expected the podGVK to have been removed") - - // cleanup test - cancel() - runWg.Wait() - require.NoError(t, runErr, "Tracker Run() failed") + t.Run(tc.name, func(t *testing.T) { + objDataFn := func() objData { + return objData{retries: tc.retries} + } + rt := newTracker(lister, false, false, false, objDataFn) + + ctx, cancel := context.WithCancel(context.Background()) + var runErr error + runWg := sync.WaitGroup{} + runWg.Add(1) + go func() { + runErr = rt.Run(ctx) + // wait for the ready tracker to stop so we don't leak state between tests. + runWg.Done() + }() + + require.Eventually(t, func() bool { + return rt.Populated() + }, timeout, tick, "waiting for RT to populated") + require.False(t, rt.Satisfied(), "tracker with retries should not be satisfied") + + // observe the sync source for readiness + rt.syncsets.Observe(&testSyncSet) + + for i := tc.retries; i > 0; i-- { + require.False(t, rt.data.Satisfied(), "data tracker should not be satisfied") + require.False(t, rt.Satisfied(), fmt.Sprintf("tracker with %d retries should not be satisfied", i)) + rt.TryCancelData(podGVK) + } + require.False(t, rt.Satisfied(), "tracker should not be satisfied") + + rt.TryCancelData(podGVK) // at this point there should no retries + require.True(t, rt.Satisfied(), "tracker with 0 retries and cancellation should be satisfied") + require.True(t, rt.data.Satisfied(), "data tracker should be satisfied") + + _, removed := rt.data.removed[podGVK] + require.True(t, removed, "expected the podGVK to have been removed") + + // cleanup test + cancel() + runWg.Wait() + require.NoError(t, runErr, "Tracker Run() failed") + }) } } diff --git a/pkg/watch/errorlist_test.go b/pkg/watch/errorlist_test.go index d17da13a5f6..5c729607743 100644 --- a/pkg/watch/errorlist_test.go +++ b/pkg/watch/errorlist_test.go @@ -25,7 +25,7 @@ func Test_WatchesError(t *testing.T) { generalErr bool }{ { - name: "gvk errors, no global", + name: "gvk errors, not general", errsToAdd: errsToAdd{ gvkErrs: []gvkErr{ {err: someErr, gvk: someGVKA}, @@ -36,7 +36,7 @@ func Test_WatchesError(t *testing.T) { generalErr: false, }, { - name: "gvk errors and global", + name: "gvk errors and general error", errsToAdd: errsToAdd{ gvkErrs: []gvkErr{ {err: someErr, gvk: someGVKA}, @@ -48,7 +48,7 @@ func Test_WatchesError(t *testing.T) { generalErr: true, }, { - name: "just global", + name: "just general error", errsToAdd: errsToAdd{ errs: []error{someErr}, }, From b660943dec48718bb38345fea842f3a38cb9891c Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 2 Nov 2023 20:54:10 +0000 Subject: [PATCH 26/42] review: remove Test_ExpectationsMgr_DeletedSyncSets Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/readiness/pruner/pruner_test.go | 136 +++--------------- .../00-namespace.yaml | 4 - .../20-syncset-1.yaml | 9 -- .../syncsets-config-disjoint/21-config.yaml | 14 -- .../syncsets-overlapping/20-syncset-1.yaml | 12 -- .../syncsets-overlapping/20-syncset-2.yaml | 12 -- .../syncsets-overlapping/20-syncset-3.yaml | 12 -- .../testdata/syncsets-resources/00-gk-ns.yaml | 4 - .../syncsets-resources/11-config.yaml | 9 -- .../syncsets-resources/15-configmap-1.yaml | 7 - .../syncsets-resources/20-syncset-1.yaml | 9 -- .../syncsets-resources/20-syncset-2.yaml | 9 -- test/testutils/applier.go | 67 --------- 13 files changed, 21 insertions(+), 283 deletions(-) delete mode 100644 pkg/readiness/pruner/testdata/syncsets-config-disjoint/00-namespace.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-config-disjoint/20-syncset-1.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-config-disjoint/21-config.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-1.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-2.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-3.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-resources/00-gk-ns.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-resources/11-config.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-resources/15-configmap-1.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml delete mode 100644 pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-2.yaml delete mode 100644 test/testutils/applier.go diff --git a/pkg/readiness/pruner/pruner_test.go b/pkg/readiness/pruner/pruner_test.go index 31e3af008c5..cc0b8214ef1 100644 --- a/pkg/readiness/pruner/pruner_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -2,13 +2,10 @@ package pruner import ( "context" - "fmt" "testing" "time" frameworksexternaldata "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" - configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" - syncsetv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/syncset/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller" @@ -22,7 +19,6 @@ import ( testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,9 +35,6 @@ const ( var cfg *rest.Config var ( - syncsetGVK = syncsetv1alpha1.GroupVersion.WithKind("SyncSet") - configGVK = configv1alpha1.GroupVersion.WithKind("Config") - configMapGVK = schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"} podGVK = schema.GroupVersionKind{Version: "v1", Kind: "Pod"} ) @@ -50,31 +43,19 @@ func TestMain(m *testing.M) { testutils.StartControlPlane(m, &cfg, 3) } -type testOptions struct { - addControllers bool - addExpectationsPruner bool - readyTrackerClient readiness.Lister -} - type testResources struct { expectationsPruner *ExpectationsPruner manager manager.Manager k8sClient client.Client } -func setupTest(ctx context.Context, t *testing.T, o testOptions) *testResources { +func setupTest(ctx context.Context, t *testing.T, readyTrackerClient readiness.Lister) *testResources { t.Helper() mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - var tracker *readiness.Tracker - var err error - if o.readyTrackerClient != nil { - tracker = readiness.NewTracker(o.readyTrackerClient, false, false, false) - } else { - tracker = readiness.NewTracker(mgr.GetAPIReader(), false, false, false) - } + tracker := readiness.NewTracker(readyTrackerClient, false, false, false) require.NoError(t, mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { return tracker.Run(ctx) })), "adding tracker to manager") @@ -100,109 +81,35 @@ func setupTest(ctx context.Context, t *testing.T, o testOptions) *testResources cm, err := cachemanager.NewCacheManager(config) require.NoError(t, err, "creating cachemanager") - if !o.addControllers { - // need to start the cachemanager if controllers are not started - // since the cachemanager is started in the controllers code. - require.NoError(t, mgr.Add(cm), "adding cachemanager as a runnable") - } else { - sw := watch.NewSwitch() - mutationSystem := mutation.NewSystem(mutation.SystemOpts{}) - - frameworksexternaldata.NewCache() - opts := controller.Dependencies{ - CFClient: testutils.SetupDataClient(t), - WatchManger: wm, - ControllerSwitch: sw, - Tracker: tracker, - ProcessExcluder: process.Get(), - MutationSystem: mutationSystem, - ExpansionSystem: expansion.NewSystem(mutationSystem), - ProviderCache: frameworksexternaldata.NewCache(), - CacheMgr: cm, - SyncEventsCh: events, - } - require.NoError(t, controller.AddToManager(mgr, &opts), "registering controllers") + sw := watch.NewSwitch() + mutationSystem := mutation.NewSystem(mutation.SystemOpts{}) + frameworksexternaldata.NewCache() + opts := controller.Dependencies{ + CFClient: testutils.SetupDataClient(t), + WatchManger: wm, + ControllerSwitch: sw, + Tracker: tracker, + ProcessExcluder: process.Get(), + MutationSystem: mutationSystem, + ExpansionSystem: expansion.NewSystem(mutationSystem), + ProviderCache: frameworksexternaldata.NewCache(), + CacheMgr: cm, + SyncEventsCh: events, } + require.NoError(t, controller.AddToManager(mgr, &opts), "registering controllers") ep := &ExpectationsPruner{ cacheMgr: cm, tracker: tracker, } - if o.addExpectationsPruner { - require.NoError(t, mgr.Add(ep), "adding expectationspruner as a runnable") - } + require.NoError(t, mgr.Add(ep), "adding expectationspruner as a runnable") return &testResources{expectationsPruner: ep, manager: mgr, k8sClient: c} } -// Test_ExpectationsMgr_DeletedSyncSets tests scenarios in which SyncSet and Config resources -// get deleted after tracker expectations have been populated and we need to reconcile -// the GVKs that are in the data client (via the cachemaanger) and the GVKs that are expected -// by the Tracker. -func Test_ExpectationsMgr_DeletedSyncSets(t *testing.T) { - tts := []struct { - name string - fixturesPath string - syncsetsToDelete []string - deleteConfig string - }{ - { - name: "delete all syncsets", - fixturesPath: "testdata/syncsets-overlapping", - syncsetsToDelete: []string{"syncset-1", "syncset-2", "syncset-3"}, - }, - { - name: "delete syncs and configs", - fixturesPath: "testdata/syncsets-config-disjoint", - syncsetsToDelete: []string{"syncset-1"}, - deleteConfig: "config", - }, - } - - for _, tt := range tts { - t.Run(tt.name, func(t *testing.T) { - ctx, cancelFunc := context.WithCancel(context.Background()) - - require.NoError(t, testutils.ApplyFixtures(tt.fixturesPath, cfg), "applying base fixtures") - - testRes := setupTest(ctx, t, testOptions{}) - - testutils.StartManager(ctx, t, testRes.manager) - - require.Eventually(t, func() bool { - return testRes.expectationsPruner.tracker.Populated() - }, timeout, tick, "waiting on tracker to populate") - - for _, name := range tt.syncsetsToDelete { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(syncsetGVK) - u.SetName(name) - - require.NoError(t, testRes.k8sClient.Delete(ctx, u), fmt.Sprintf("deleting syncset %s", name)) - } - if tt.deleteConfig != "" { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(configGVK) - u.SetNamespace("gatekeeper-system") - u.SetName(tt.deleteConfig) - - require.NoError(t, testRes.k8sClient.Delete(ctx, u), fmt.Sprintf("deleting config %s", tt.deleteConfig)) - } - - testRes.expectationsPruner.pruneUnwatchedGVKs() - - require.Eventually(t, func() bool { - return testRes.expectationsPruner.tracker.Satisfied() - }, timeout, tick, "waiting on tracker to get satisfied") - - cancelFunc() - }) - } -} - -// Test_ExpectationsMgr_missedInformers verifies that the pruner can handle a scenario +// Test_ExpectationsPruner_missedInformers verifies that the pruner can handle a scenario // where the readiness tracker's state will never match the informer cache events. -func Test_ExpectationsMgr_missedInformers(t *testing.T) { +func Test_ExpectationsPruner_missedInformers(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) // Set up one data store for readyTracker: @@ -213,8 +120,7 @@ func Test_ExpectationsMgr_missedInformers(t *testing.T) { fakes.UnstructuredFor(podGVK, "", "pod1-name"), fakes.UnstructuredFor(configMapGVK, "", "cm1-name"), ).Build() - - testRes := setupTest(ctx, t, testOptions{readyTrackerClient: lister, addExpectationsPruner: true, addControllers: true}) + testRes := setupTest(ctx, t, lister) // Set up another store for the controllers and watchManager syncsetA := fakes.SyncSetFor("syncset-a", []schema.GroupVersionKind{podGVK}) diff --git a/pkg/readiness/pruner/testdata/syncsets-config-disjoint/00-namespace.yaml b/pkg/readiness/pruner/testdata/syncsets-config-disjoint/00-namespace.yaml deleted file mode 100644 index 52a227623e5..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-config-disjoint/00-namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: "gatekeeper-system" diff --git a/pkg/readiness/pruner/testdata/syncsets-config-disjoint/20-syncset-1.yaml b/pkg/readiness/pruner/testdata/syncsets-config-disjoint/20-syncset-1.yaml deleted file mode 100644 index 530ac2d2130..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-config-disjoint/20-syncset-1.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: syncset.gatekeeper.sh/v1alpha1 -kind: SyncSet -metadata: - name: syncset-1 -spec: - gvks: - - group: "" - version: "v1" - kind: "Pod" diff --git a/pkg/readiness/pruner/testdata/syncsets-config-disjoint/21-config.yaml b/pkg/readiness/pruner/testdata/syncsets-config-disjoint/21-config.yaml deleted file mode 100644 index b5fd7165ef4..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-config-disjoint/21-config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: config.gatekeeper.sh/v1alpha1 -kind: Config -metadata: - name: config - namespace: "gatekeeper-system" -spec: - sync: - syncOnly: - - group: "" - version: "v1" - kind: "Namespace" - - group: "" - version: "v1" - kind: "ConfigMap" diff --git a/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-1.yaml b/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-1.yaml deleted file mode 100644 index 3f1f8450e3c..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-1.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: syncset.gatekeeper.sh/v1alpha1 -kind: SyncSet -metadata: - name: syncset-1 -spec: - gvks: - - group: "" - version: "v1" - kind: "Pod" - - group: "" - version: "v1" - kind: "ConfigMap" diff --git a/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-2.yaml b/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-2.yaml deleted file mode 100644 index b18d51248d9..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: syncset.gatekeeper.sh/v1alpha1 -kind: SyncSet -metadata: - name: syncset-2 -spec: - gvks: - - group: "" - version: "v1" - kind: "Namespace" - - group: "" - version: "v1" - kind: "ConfigMap" diff --git a/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-3.yaml b/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-3.yaml deleted file mode 100644 index 7587d8ce22a..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-overlapping/20-syncset-3.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: syncset.gatekeeper.sh/v1alpha1 -kind: SyncSet -metadata: - name: syncset-3 -spec: - gvks: - - group: "" - version: "v1" - kind: "Pod" - - group: "" - version: "v1" - kind: "Namespace" diff --git a/pkg/readiness/pruner/testdata/syncsets-resources/00-gk-ns.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/00-gk-ns.yaml deleted file mode 100644 index 52a227623e5..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-resources/00-gk-ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: "gatekeeper-system" diff --git a/pkg/readiness/pruner/testdata/syncsets-resources/11-config.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/11-config.yaml deleted file mode 100644 index 357b9c6a870..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-resources/11-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: config.gatekeeper.sh/v1alpha1 -kind: Config -metadata: - name: config - namespace: "gatekeeper-system" -spec: - match: - - excludedNamespaces: ["kube-*"] - processes: ["*"] diff --git a/pkg/readiness/pruner/testdata/syncsets-resources/15-configmap-1.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/15-configmap-1.yaml deleted file mode 100644 index 25a3a17b343..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-resources/15-configmap-1.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: "some-config-map" - namespace: "default" -data: - foo: bar diff --git a/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml deleted file mode 100644 index 9c795bb7039..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-1.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: syncset.gatekeeper.sh/v1alpha1 -kind: SyncSet -metadata: - name: syncset-1 -spec: - gvks: - - group: "" - version: "v1" - kind: "Namespace" diff --git a/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-2.yaml b/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-2.yaml deleted file mode 100644 index b955227a477..00000000000 --- a/pkg/readiness/pruner/testdata/syncsets-resources/20-syncset-2.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: syncset.gatekeeper.sh/v1alpha1 -kind: SyncSet -metadata: - name: syncset-2 -spec: - gvks: - - group: "" - version: "v1" - kind: "UnknownPod" diff --git a/test/testutils/applier.go b/test/testutils/applier.go deleted file mode 100644 index c3a3094cec0..00000000000 --- a/test/testutils/applier.go +++ /dev/null @@ -1,67 +0,0 @@ -package testutils - -import ( - "context" - "fmt" - "os" - "path/filepath" - "sort" - - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/yaml" -) - -// Applies fixture YAMLs directly under the provided path in alpha-sorted order. -// Does not crawl the directory, instead it only looks at the files present at path. -func ApplyFixtures(path string, cfg *rest.Config) error { - files, err := os.ReadDir(path) - if err != nil { - return fmt.Errorf("reading path %s: %w", path, err) - } - - c, err := client.New(cfg, client.Options{}) - if err != nil { - return fmt.Errorf("creating client: %w", err) - } - - sorted := make([]string, 0, len(files)) - for _, entry := range files { - if entry.IsDir() { - continue - } - sorted = append(sorted, entry.Name()) - } - sort.StringSlice(sorted).Sort() - - for _, entry := range sorted { - b, err := os.ReadFile(filepath.Join(path, entry)) - if err != nil { - return fmt.Errorf("reading file %s: %w", entry, err) - } - - desired := unstructured.Unstructured{} - if err := yaml.Unmarshal(b, &desired); err != nil { - return fmt.Errorf("parsing file %s: %w", entry, err) - } - - u := unstructured.Unstructured{} - u.SetGroupVersionKind(desired.GroupVersionKind()) - u.SetName(desired.GetName()) - u.SetNamespace(desired.GetNamespace()) - _, err = controllerutil.CreateOrUpdate(context.Background(), c, &u, func() error { - resourceVersion := u.GetResourceVersion() - desired.DeepCopyInto(&u) - u.SetResourceVersion(resourceVersion) - - return nil - }) - if err != nil { - return fmt.Errorf("creating %v %s: %w", u.GroupVersionKind(), u.GetName(), err) - } - } - - return nil -} From 79dce07492aa5a29325448fd03fafa7b3a79f17f Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 2 Nov 2023 22:55:37 +0000 Subject: [PATCH 27/42] refactor: agg err, restore agg - don't error in agg - restore agg in cm on watch error Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/aggregator/aggregator.go | 38 +++++--------- .../aggregator/aggregator_test.go | 50 +++++++------------ pkg/cachemanager/cachemanager.go | 25 ++++++---- pkg/cachemanager/cachemanager_test.go | 2 +- .../cachemanager_integration_test.go | 6 +-- 5 files changed, 49 insertions(+), 72 deletions(-) diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index 549e57dfc1d..8e9e4e4fcec 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -1,7 +1,6 @@ package aggregator import ( - "fmt" gosync "sync" "k8s.io/apimachinery/pkg/runtime/schema" @@ -49,30 +48,25 @@ func (b *GVKAgreggator) IsPresent(gvk schema.GroupVersionKind) bool { // Remove deletes any associations that Key k has in the GVKAggregator. // For any GVK in the association k --> [GVKs], we also delete any associations // between the GVK and the Key k stored in the reverse map. -func (b *GVKAgreggator) Remove(k Key) error { +func (b *GVKAgreggator) Remove(k Key) { b.mu.Lock() defer b.mu.Unlock() gvks, found := b.store[k] if !found { - return nil + return } - if err := b.pruneReverseStore(gvks, k); err != nil { - return err - } + b.pruneReverseStore(gvks, k) delete(b.store, k) - return nil } // Upsert stores an association between Key k and the list of GVKs // and also the reverse associatoin between each GVK passed in and Key k. // Any old associations are dropped, unless they are included in the new list of // GVKs. -// It errors out if there is an internal issue with remove the reverse Key links -// for any GVKs that are being dropped as part of this Upsert call. -func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) error { +func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) { b.mu.Lock() defer b.mu.Unlock() @@ -80,15 +74,13 @@ func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) error { if found { // gvksToRemove contains old GKVs that are not included in the new gvks list gvksToRemove := unreferencedOldGVKsToPrune(gvks, oldGVKs) - if err := b.pruneReverseStore(gvksToRemove, k); err != nil { - return fmt.Errorf("failed to prune entries on upsert: %w", err) - } + b.pruneReverseStore(gvksToRemove, k) } // protect against empty inputs gvksSet := makeSet(gvks) if len(gvksSet) == 0 { - return nil + return } b.store[k] = gvksSet @@ -99,19 +91,17 @@ func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) error { } b.reverseStore[gvk][k] = struct{}{} } - - return nil } // List returnes the gvk set for a given Key. -func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} { +func (b *GVKAgreggator) List(k Key) []schema.GroupVersionKind { b.mu.RLock() defer b.mu.RUnlock() v := b.store[k] - cpy := make(map[schema.GroupVersionKind]struct{}, len(v)) - for key, value := range v { - cpy[key] = value + cpy := []schema.GroupVersionKind{} + for key := range v { + cpy = append(cpy, key) } return cpy } @@ -149,13 +139,13 @@ func (b *GVKAgreggator) GVKs() []schema.GroupVersionKind { return allGVKs } -func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struct{}, k Key) error { +func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struct{}, k Key) { for gvk := range gvks { keySet, found := b.reverseStore[gvk] - if !found || len(keySet) == 0 { + if !found { // this should not happen if we keep the two maps well defined // but let's be defensive nonetheless. - return fmt.Errorf("internal aggregator error: gvks stores are corrupted for key: %s", k) + return } delete(keySet, k) @@ -167,8 +157,6 @@ func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struc b.reverseStore[gvk] = keySet } } - - return nil } func makeSet(gvks []schema.GroupVersionKind) map[schema.GroupVersionKind]struct{} { diff --git a/pkg/cachemanager/aggregator/aggregator_test.go b/pkg/cachemanager/aggregator/aggregator_test.go index af8bc4fd7d2..4725ce20d70 100644 --- a/pkg/cachemanager/aggregator/aggregator_test.go +++ b/pkg/cachemanager/aggregator/aggregator_test.go @@ -204,7 +204,7 @@ func Test_GVKAggregator_Upsert(t *testing.T) { agg := NewGVKAggregator() for _, keyGVKs := range tt.keyGVKs { - require.NoError(t, agg.Upsert(keyGVKs.key, keyGVKs.gvks)) + agg.Upsert(keyGVKs.key, keyGVKs.gvks) } // require all gvks added to be present in the aggregator @@ -218,8 +218,7 @@ func Test_GVKAgreggator_Remove(t *testing.T) { t.Run("Remove on empty aggregator", func(t *testing.T) { b := NewGVKAggregator() key := Key{Source: "testSource", ID: "testID"} - - require.NoError(t, b.Remove(key)) + b.Remove(key) }) t.Run("Remove non-existing key", func(t *testing.T) { @@ -227,50 +226,35 @@ func Test_GVKAgreggator_Remove(t *testing.T) { key1 := Key{Source: syncset, ID: "testID1"} key2 := Key{Source: configsync, ID: "testID2"} gvks := []schema.GroupVersionKind{g1v1k1, g1v1k2} - require.NoError(t, b.Upsert(key1, gvks)) - - require.NoError(t, b.Remove(key2)) + b.Upsert(key1, gvks) + b.Remove(key2) }) t.Run("Remove existing key and verify reverseStore", func(t *testing.T) { b := NewGVKAggregator() key1 := Key{Source: syncset, ID: "testID"} gvks := []schema.GroupVersionKind{g1v1k1, g1v1k2} - require.NoError(t, b.Upsert(key1, gvks)) - - require.NoError(t, b.Remove(key1)) + b.Upsert(key1, gvks) + b.Remove(key1) for _, gvk := range gvks { require.False(t, b.IsPresent(gvk)) } }) - t.Run("Remove existing 1 of existing keys referencing GVKs", func(t *testing.T) { + t.Run("Remove 1 of existing keys referencing GVKs", func(t *testing.T) { b := NewGVKAggregator() key1 := Key{Source: syncset, ID: "testID"} key2 := Key{Source: configsync, ID: "testID"} gvks := []schema.GroupVersionKind{g1v1k1, g1v1k2} - require.NoError(t, b.Upsert(key1, gvks)) - require.NoError(t, b.Upsert(key2, gvks)) - - require.NoError(t, b.Remove(key1)) + b.Upsert(key1, gvks) + b.Upsert(key2, gvks) + b.Remove(key1) for _, gvk := range gvks { require.True(t, b.IsPresent(gvk)) } }) - - t.Run("corrupted GVKAggregator error", func(t *testing.T) { - b := NewGVKAggregator() - key := Key{Source: syncset, ID: "testID"} - - // simulate a corrupted aggreagator - require.NoError(t, b.Upsert(key, []schema.GroupVersionKind{g1v1k1})) - b.reverseStore[g1v1k1] = map[Key]struct{}{} - - err2 := b.Remove(key) - require.EqualError(t, err2, fmt.Sprintf("internal aggregator error: gvks stores are corrupted for key: {%s %s}", key.Source, key.ID)) - }) } func Test_GVKAggregator_ListNotShared(t *testing.T) { @@ -333,7 +317,7 @@ func Test_GVKAggregator_ListNotShared(t *testing.T) { agg := NewGVKAggregator() for _, keyGVKs := range tt.keyGVKs { - require.NoError(t, agg.Upsert(keyGVKs.key, keyGVKs.gvks)) + agg.Upsert(keyGVKs.key, keyGVKs.gvks) } require.ElementsMatch(t, tt.expectedNotShared, agg.ListNotShared(syncSetKey)) @@ -355,8 +339,8 @@ func Test_GVKAggreggator_E2E(t *testing.T) { gvksKind1 := []schema.GroupVersionKind{g1v1k1, g2v1k1, g2v2k1} gvksKind2 := []schema.GroupVersionKind{g1v1k2, g2v1k2, g2v2k2} - require.NoError(t, b.Upsert(key1, gvksKind1)) // key1 now has: g1v1k1, g2v1k1, g2v2k1 - require.NoError(t, b.Upsert(key2, gvksKind2)) // key2 now has: g1v1k2, g2v1k2, g2v2k2 + b.Upsert(key1, gvksKind1) // key1 now has: g1v1k1, g2v1k1, g2v2k1 + b.Upsert(key2, gvksKind2) // key2 now has: g1v1k2, g2v1k2, g2v2k2 // require that every gvk that was just added to be present gvksThatShouldBeTracked := map[schema.GroupVersionKind]interface{}{ @@ -372,8 +356,8 @@ func Test_GVKAggreggator_E2E(t *testing.T) { gvksVersion2 := []schema.GroupVersionKind{g1v2k1, g1v2k2, g2v2k1, g2v2k2} // overlaps key1 with g2v2k1; overlaps key2 with g2v2k2 // new upserts - require.NoError(t, b.Upsert(key1, gvksVersion1)) // key1 no longer associates g2v2k1, but key2 does - require.NoError(t, b.Upsert(key2, gvksVersion2)) // key2 no longer associaates g1v1k2, but key1 does + b.Upsert(key1, gvksVersion1) // key1 no longer associates g2v2k1, but key2 does + b.Upsert(key2, gvksVersion2) // key2 no longer associaates g1v1k2, but key1 does // require that every gvk that was just added now and before to be present for _, gvk := range append(append([]schema.GroupVersionKind{}, gvksVersion1...), gvksVersion2...) { @@ -386,7 +370,7 @@ func Test_GVKAggreggator_E2E(t *testing.T) { // key1 has: g1v1k1, g2v1k1, g1v1k2, g2v1k2 // key2 has: g2v2k2, g1v2k1, g1v2k2, g2v2k1 // now remove key1 - require.NoError(t, b.Remove(key1)) + b.Remove(key1) // untrack gvks that shouldn't exist in the for _, gvk := range []schema.GroupVersionKind{g1v1k1, g2v1k1, g1v1k2, g2v1k2} { @@ -403,7 +387,7 @@ func Test_GVKAggreggator_E2E(t *testing.T) { // overwrite key2 gvksGroup3 := []schema.GroupVersionKind{g3v1k1, g3v1k2} - require.NoError(t, b.Upsert(key2, gvksGroup3)) + b.Upsert(key2, gvksGroup3) // require all previously added gvks to not be present: testPresenceForGVK(t, false, b, gvksKind1...) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 91337a04430..848b6c2c259 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -116,14 +116,11 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke c.mu.Lock() defer c.mu.Unlock() + currentGVKsForKey := c.gvksToSync.List(sourceKey) if len(newGVKs) > 0 { - if err := c.gvksToSync.Upsert(sourceKey, newGVKs); err != nil { - return fmt.Errorf("internal error adding source: %w", err) - } + c.gvksToSync.Upsert(sourceKey, newGVKs) } else { - if err := c.gvksToSync.Remove(sourceKey); err != nil { - return fmt.Errorf("internal error removing source: %w", err) - } + c.gvksToSync.Remove(sourceKey) } // as a result of upserting the new gvks for the source key, some gvks @@ -144,6 +141,12 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke for _, g := range gvksToTryCancel { c.tracker.TryCancelData(g) } + + // restore the gvk aggregator's key before sending out the error so clients can retry + if len(currentGVKsForKey) > 0 { + c.gvksToSync.Upsert(sourceKey, currentGVKsForKey) + } + return fmt.Errorf("error establishing watches: %w", err) } @@ -201,13 +204,17 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke c.mu.Lock() defer c.mu.Unlock() + currentGVKsForKey := c.gvksToSync.List(sourceKey) gvksNotShared := c.gvksToSync.ListNotShared(sourceKey) - if err := c.gvksToSync.Remove(sourceKey); err != nil { - return fmt.Errorf("internal error removing source: %w", err) - } + c.gvksToSync.Remove(sourceKey) err := c.replaceWatchSet(ctx) if general, failedGVKs := interpretErr(err, gvksNotShared); general || len(failedGVKs) > 0 { + // restore the gvk aggregator before sending out the error so clients can retry + if len(currentGVKsForKey) > 0 { + c.gvksToSync.Upsert(sourceKey, currentGVKsForKey) + } + return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index d000f702f3d..4b77567b9e0 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -603,7 +603,7 @@ func TestCacheManager_RemoveSource(t *testing.T) { cm, ctx := makeCacheManager(t) // seed the cachemanager internals for _, s := range tc.existingSources { - require.NoError(t, cm.gvksToSync.Upsert(s.key, s.gvks)) + cm.gvksToSync.Upsert(s.key, s.gvks) } for _, source := range tc.sourcesToRemove { diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index 7618c62bf45..c67526af83a 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -210,12 +210,10 @@ func TestCacheManager_concurrent(t *testing.T) { agg.IsPresent(configMapGVK) gvks := agg.List(syncSourceOne) require.Len(t, gvks, 1) - _, foundConfigMap := gvks[configMapGVK] - require.True(t, foundConfigMap) + require.Equal(t, configMapGVK, gvks[0]) gvks = agg.List(syncSourceTwo) require.Len(t, gvks, 1) - _, foundPod := gvks[podGVK] - require.True(t, foundPod) + require.Equal(t, podGVK, gvks[0]) // do a final remove and expect the cache to clear require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) From 09842ff4f8233ef342fb9b361033b9c67bdb3f9b Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 3 Nov 2023 21:45:30 +0000 Subject: [PATCH 28/42] revert: restore agg Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/aggregator/aggregator.go | 21 ------ .../aggregator/aggregator_test.go | 68 ------------------- pkg/cachemanager/cachemanager.go | 16 +---- 3 files changed, 1 insertion(+), 104 deletions(-) diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index 8e9e4e4fcec..d573f368881 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -106,27 +106,6 @@ func (b *GVKAgreggator) List(k Key) []schema.GroupVersionKind { return cpy } -// List returnes the gvks for a given Key that are not referenced by any other Key. -func (b *GVKAgreggator) ListNotShared(k Key) []schema.GroupVersionKind { - b.mu.RLock() - defer b.mu.RUnlock() - - gvks := b.store[k] - gvksToReturn := []schema.GroupVersionKind{} - for gvk := range gvks { - keys, ok := b.reverseStore[gvk] - if !ok { - continue - } - - if len(keys) == 1 { // by definition this Key k is the only one referencing this gvk - gvksToReturn = append(gvksToReturn, gvk) - } - } - - return gvksToReturn -} - // GVKs returns a list of all of the schema.GroupVersionKind that are aggregated. func (b *GVKAgreggator) GVKs() []schema.GroupVersionKind { b.mu.RLock() diff --git a/pkg/cachemanager/aggregator/aggregator_test.go b/pkg/cachemanager/aggregator/aggregator_test.go index 4725ce20d70..17a6e51f103 100644 --- a/pkg/cachemanager/aggregator/aggregator_test.go +++ b/pkg/cachemanager/aggregator/aggregator_test.go @@ -257,74 +257,6 @@ func Test_GVKAgreggator_Remove(t *testing.T) { }) } -func Test_GVKAggregator_ListNotShared(t *testing.T) { - syncSetKey := Key{Source: syncset, ID: "foo"} // we will list this key - configKey := Key{Source: configsync, ID: "foo"} - - tests := []struct { - name string - keyGVKs []keyedGVKs - - expectedNotShared []schema.GroupVersionKind - }{ - { - name: "all gvks shared", - keyGVKs: []keyedGVKs{ - { - key: syncSetKey, - gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}, - }, - { - key: configKey, - gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}, - }, - }, - expectedNotShared: []schema.GroupVersionKind{}, - }, - { - name: "no gvks shared", - keyGVKs: []keyedGVKs{ - { - key: syncSetKey, - gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}, - }, - { - key: configKey, - gvks: []schema.GroupVersionKind{g2v1k1, g2v1k2}, - }, - }, - expectedNotShared: []schema.GroupVersionKind{g1v1k1, g1v1k2}, - }, - { - name: "some gvks shared", - keyGVKs: []keyedGVKs{ - { - key: syncSetKey, - gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}, - }, - { - key: configKey, - gvks: []schema.GroupVersionKind{g1v1k1, g2v1k2}, - }, - }, - expectedNotShared: []schema.GroupVersionKind{g1v1k2}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt := tt - agg := NewGVKAggregator() - - for _, keyGVKs := range tt.keyGVKs { - agg.Upsert(keyGVKs.key, keyGVKs.gvks) - } - - require.ElementsMatch(t, tt.expectedNotShared, agg.ListNotShared(syncSetKey)) - }) - } -} - // Test_GVKAggreggator_E2E is a test that: // - Upserts two sources with different GVKs // - Upserts the two sources with some overlapping GVKs diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 848b6c2c259..b7a46078609 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -116,7 +116,6 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke c.mu.Lock() defer c.mu.Unlock() - currentGVKsForKey := c.gvksToSync.List(sourceKey) if len(newGVKs) > 0 { c.gvksToSync.Upsert(sourceKey, newGVKs) } else { @@ -142,11 +141,6 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke c.tracker.TryCancelData(g) } - // restore the gvk aggregator's key before sending out the error so clients can retry - if len(currentGVKsForKey) > 0 { - c.gvksToSync.Upsert(sourceKey, currentGVKsForKey) - } - return fmt.Errorf("error establishing watches: %w", err) } @@ -204,17 +198,9 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke c.mu.Lock() defer c.mu.Unlock() - currentGVKsForKey := c.gvksToSync.List(sourceKey) - gvksNotShared := c.gvksToSync.ListNotShared(sourceKey) - c.gvksToSync.Remove(sourceKey) err := c.replaceWatchSet(ctx) - if general, failedGVKs := interpretErr(err, gvksNotShared); general || len(failedGVKs) > 0 { - // restore the gvk aggregator before sending out the error so clients can retry - if len(currentGVKsForKey) > 0 { - c.gvksToSync.Upsert(sourceKey, currentGVKsForKey) - } - + if general, _ := interpretErr(err, []schema.GroupVersionKind{}); general { return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) } From 3d077787c4d3c466561687063dd4db7abe4ec2bd Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 3 Nov 2023 21:53:37 +0000 Subject: [PATCH 29/42] revert: infer podGVK Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/readiness/ready_tracker_unit_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/readiness/ready_tracker_unit_test.go b/pkg/readiness/ready_tracker_unit_test.go index 39bfa9ca3a1..11d28213036 100644 --- a/pkg/readiness/ready_tracker_unit_test.go +++ b/pkg/readiness/ready_tracker_unit_test.go @@ -28,6 +28,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/stretchr/testify/require" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -63,7 +64,7 @@ var ( }, } - podGVK = testSyncSet.Spec.GVKs[0].ToGroupVersionKind() + podGVK = schema.GroupVersionKind{Version: "v1", Kind: "Pod"} ) var convertedTemplate v1beta1.ConstraintTemplate From ccf21600ecc35d4fe04da8f36e5053081ec05d50 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 3 Nov 2023 22:21:20 +0000 Subject: [PATCH 30/42] add RemoveGVKErr Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 2 +- pkg/cachemanager/cachemanager_test.go | 10 +++++++++- pkg/watch/errorlist.go | 19 +++++++++++++------ pkg/watch/errorlist_test.go | 12 +++++++++--- pkg/watch/manager.go | 2 +- 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index b7a46078609..fdcb8429eca 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -178,7 +178,7 @@ func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.Group } failedGvks := watch.NewSet() - failedGvks.Add(f.FailingGVKs()...) + failedGvks.Add(f.FailingGVKsToAdd()...) sourceGVKSet := watch.NewSet() sourceGVKSet.Add(gvks...) diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 4b77567b9e0..6d7bfd186fe 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -622,8 +622,10 @@ func Test_interpretErr(t *testing.T) { gvk1Err := watch.NewErrorList() gvk1Err.AddGVKErr(gvk1, someErr) genErr := watch.NewErrorList() - genErr.Add(someErr) + genErr.Err(someErr) genErr.AddGVKErr(gvk1, someErr) + gvk2Err := watch.NewErrorList() + gvk2Err.RemoveGVKErr(gvk2, someErr) cases := []struct { name string @@ -647,6 +649,12 @@ func Test_interpretErr(t *testing.T) { inputErr: gvk1Err, inputGVK: []schema.GroupVersionKind{gvk2}, }, + { + name: "intersection exists but we are removing the gvk", + inputErr: gvk2Err, + inputGVK: []schema.GroupVersionKind{gvk2}, + expectedFailingGVKs: []schema.GroupVersionKind{}, + }, { name: "non-watchmanager error reports general error with no GVKs", inputErr: fmt.Errorf("some err: %w", someErr), diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index 3a37d4d9525..e9ba3c9c367 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -24,8 +24,9 @@ import ( ) type gvkErr struct { - err error - gvk schema.GroupVersionKind + err error + gvk schema.GroupVersionKind + isRemove bool } func (w gvkErr) String() string { @@ -63,11 +64,12 @@ func (e *ErrorList) Error() string { return builder.String() } -func (e *ErrorList) FailingGVKs() []schema.GroupVersionKind { +// Return gvks for which there were errors adding watches. +func (e *ErrorList) FailingGVKsToAdd() []schema.GroupVersionKind { gvks := []schema.GroupVersionKind{} for _, err := range e.errs { var gvkErr gvkErr - if errors.As(err, &gvkErr) { + if errors.As(err, &gvkErr) && !gvkErr.isRemove { gvks = append(gvks, gvkErr.gvk) } } @@ -80,16 +82,21 @@ func (e *ErrorList) HasGeneralErr() bool { } // adds a non gvk specific error to the list. -func (e *ErrorList) Add(err error) { +func (e *ErrorList) Err(err error) { e.errs = append(e.errs, err) e.hasGeneralErr = true } -// adds a gvk specific error to the list. +// adds a gvk specific error for failing to add a gvk watch to the list. func (e *ErrorList) AddGVKErr(gvk schema.GroupVersionKind, err error) { e.errs = append(e.errs, gvkErr{gvk: gvk, err: err}) } +// adds a gvk specific error for failing to remove a gvk watch to the list. +func (e *ErrorList) RemoveGVKErr(gvk schema.GroupVersionKind, err error) { + e.errs = append(e.errs, gvkErr{gvk: gvk, err: err, isRemove: true}) +} + func (e *ErrorList) Size() int { return len(e.errs) } diff --git a/pkg/watch/errorlist_test.go b/pkg/watch/errorlist_test.go index 5c729607743..555cc1c5ccf 100644 --- a/pkg/watch/errorlist_test.go +++ b/pkg/watch/errorlist_test.go @@ -12,6 +12,7 @@ func Test_WatchesError(t *testing.T) { someErr := errors.New("some err") someGVKA := schema.GroupVersionKind{Group: "a", Version: "b", Kind: "c"} someGVKB := schema.GroupVersionKind{Group: "x", Version: "y", Kind: "z"} + someGVKC := schema.GroupVersionKind{Group: "m", Version: "n", Kind: "o"} type errsToAdd struct { errs []error @@ -41,6 +42,7 @@ func Test_WatchesError(t *testing.T) { gvkErrs: []gvkErr{ {err: someErr, gvk: someGVKA}, {err: someErr, gvk: someGVKB}, + {err: someErr, gvk: someGVKC, isRemove: true}, // this one should not show up in FailingGVKsToAdd }, errs: []error{someErr, someErr}, }, @@ -60,13 +62,17 @@ func Test_WatchesError(t *testing.T) { t.Run(tc.name, func(t *testing.T) { er := NewErrorList() for _, gvkErr := range tc.errsToAdd.gvkErrs { - er.AddGVKErr(gvkErr.gvk, gvkErr.err) + if gvkErr.isRemove { + er.RemoveGVKErr(gvkErr.gvk, gvkErr.err) + } else { + er.AddGVKErr(gvkErr.gvk, gvkErr.err) + } } for _, err := range tc.errsToAdd.errs { - er.Add(err) + er.Err(err) } - require.ElementsMatch(t, tc.expectedGVKs, er.FailingGVKs()) + require.ElementsMatch(t, tc.expectedGVKs, er.FailingGVKsToAdd()) require.Equal(t, tc.generalErr, er.HasGeneralErr()) }) } diff --git a/pkg/watch/manager.go b/pkg/watch/manager.go index c1e133c8947..0bff23d72c0 100644 --- a/pkg/watch/manager.go +++ b/pkg/watch/manager.go @@ -263,7 +263,7 @@ func (wm *Manager) replaceWatches(ctx context.Context, r *Registrar) error { continue } if err := wm.doRemoveWatch(ctx, r, gvk); err != nil { - errlist.AddGVKErr(gvk, fmt.Errorf("removing watch for %+v %w", gvk, err)) + errlist.RemoveGVKErr(gvk, fmt.Errorf("removing watch for %+v %w", gvk, err)) } } From 449b39ac94436c5efbf26d350bb9dd30181f98a5 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Sat, 4 Nov 2023 00:16:51 +0000 Subject: [PATCH 31/42] account for dangling watches Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 62 +++++++++++++++++---------- pkg/cachemanager/cachemanager_test.go | 23 +++++----- pkg/watch/errorlist.go | 12 ++++++ 3 files changed, 65 insertions(+), 32 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index fdcb8429eca..89d426c6e01 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -52,6 +52,9 @@ type CacheManager struct { needToList bool gvksToDeleteFromCache *watch.Set excluderChanged bool + danglingWatches bool + forceWipe bool + // mu guards access to any of the fields above mu sync.RWMutex @@ -127,7 +130,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // in the manageCache loop. err := c.replaceWatchSet(ctx) - if general, failedGVKs := interpretErr(err, newGVKs); len(failedGVKs) > 0 || general { + if general, failedGVKs, _ := interpretErr(err, newGVKs); len(failedGVKs) > 0 || general { var gvksToTryCancel []schema.GroupVersionKind if general { // if the err is general, assume all gvks need TryCancel because of some @@ -154,43 +157,48 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet.Add(c.gvksToSync.GVKs()...) c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) - var innerError error - c.watchedSet.Replace(newWatchSet, func() { - // *Note the following steps are not transactional with respect to admission control + if c.danglingWatches || !c.watchedSet.Equals(newWatchSet) { + var err error + c.watchedSet.Replace(newWatchSet, func() { + // *Note the following steps are not transactional with respect to admission control - // Important: dynamic watches update must happen *after* updating our watchSet. - // Otherwise, the sync controller will drop events for the newly watched kinds. - innerError = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) - }) + // Important: dynamic watches update must happen *after* updating our watchSet. + // Otherwise, the sync controller will drop events for the newly watched kinds. + err = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) + }) + + return err + } - return innerError + return nil } // interpret if the err received is general or whether it is specific to the provided GVKs. -func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind) { +// also returns whether there were any errors removing watches. +func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind, bool) { if e == nil { - return false, nil + return false, nil, false } f := watch.NewErrorList() if !errors.As(e, &f) || f.HasGeneralErr() { - return true, nil + return true, nil, false } - failedGvks := watch.NewSet() - failedGvks.Add(f.FailingGVKsToAdd()...) + failedGvksToAdd := watch.NewSet() + failedGvksToAdd.Add(f.FailingGVKsToAdd()...) sourceGVKSet := watch.NewSet() sourceGVKSet.Add(gvks...) - common := failedGvks.Intersection(sourceGVKSet) + common := failedGvksToAdd.Intersection(sourceGVKSet) if common.Size() > 0 { - return false, common.Items() + return false, common.Items(), f.HasFailingGVKsToRemove() } // this error is not about the gvks in this request // but we still log it for visibility log.V(logging.DebugLevel).Info("encountered unrelated error when replacing watch set", "error", e) - return false, nil + return false, nil, f.HasFailingGVKsToRemove() } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. @@ -199,10 +207,7 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke defer c.mu.Unlock() c.gvksToSync.Remove(sourceKey) - err := c.replaceWatchSet(ctx) - if general, _ := interpretErr(err, []schema.GroupVersionKind{}); general { - return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) - } + // watch removal will happen async in manageCache() return nil } @@ -369,6 +374,18 @@ func (c *CacheManager) manageCache(ctx context.Context) { c.mu.Lock() defer c.mu.Unlock() + // first make sure there is no drift between c.gvksToSync and watch manager + err := c.replaceWatchSet(ctx) + _, _, danglingWatches := interpretErr(err, []schema.GroupVersionKind{}) + + // if there were dangling watches previously but not anymore, + // force a cache wipe so we don't track unwanted resources. + if c.danglingWatches && !danglingWatches { + c.forceWipe = true + } + c.danglingWatches = danglingWatches + + // perform a wipe if we removed gvks or the excluder changed c.wipeCacheIfNeeded(ctx) if !c.needToList { @@ -462,13 +479,14 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro func (c *CacheManager) wipeCacheIfNeeded(ctx context.Context) { // remove any gvks not needing to be synced anymore // or re evaluate all if the excluder changed. - if c.gvksToDeleteFromCache.Size() > 0 || c.excluderChanged { + if c.gvksToDeleteFromCache.Size() > 0 || c.excluderChanged || c.forceWipe { if err := c.wipeData(ctx); err != nil { log.Error(err, "internal: error wiping cache") return } c.gvksToDeleteFromCache = watch.NewSet() + c.forceWipe = false c.excluderChanged = false c.needToList = true } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 6d7bfd186fe..151816827c9 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -628,11 +628,12 @@ func Test_interpretErr(t *testing.T) { gvk2Err.RemoveGVKErr(gvk2, someErr) cases := []struct { - name string - inputErr error - inputGVK []schema.GroupVersionKind - expectedFailingGVKs []schema.GroupVersionKind - expectGeneral bool + name string + inputErr error + inputGVK []schema.GroupVersionKind + expectedFailingGVKs []schema.GroupVersionKind + expectGeneral bool + expectDanglingWatches bool }{ { name: "nil err", @@ -650,10 +651,11 @@ func Test_interpretErr(t *testing.T) { inputGVK: []schema.GroupVersionKind{gvk2}, }, { - name: "intersection exists but we are removing the gvk", - inputErr: gvk2Err, - inputGVK: []schema.GroupVersionKind{gvk2}, - expectedFailingGVKs: []schema.GroupVersionKind{}, + name: "intersection exists but we are removing the gvk", + inputErr: gvk2Err, + inputGVK: []schema.GroupVersionKind{gvk2}, + expectedFailingGVKs: []schema.GroupVersionKind{}, + expectDanglingWatches: true, }, { name: "non-watchmanager error reports general error with no GVKs", @@ -671,10 +673,11 @@ func Test_interpretErr(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - general, gvks := interpretErr(tc.inputErr, tc.inputGVK) + general, gvks, danglingWatches := interpretErr(tc.inputErr, tc.inputGVK) require.Equal(t, tc.expectGeneral, general) require.ElementsMatch(t, gvks, tc.expectedFailingGVKs) + require.Equal(t, tc.expectDanglingWatches, danglingWatches) }) } } diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index e9ba3c9c367..62e86202634 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -77,6 +77,18 @@ func (e *ErrorList) FailingGVKsToAdd() []schema.GroupVersionKind { return gvks } +// Return gvks for which there were errors removing watches. +func (e *ErrorList) HasFailingGVKsToRemove() bool { + for _, err := range e.errs { + var gvkErr gvkErr + if errors.As(err, &gvkErr) && gvkErr.isRemove { + return true + } + } + + return false +} + func (e *ErrorList) HasGeneralErr() bool { return e.hasGeneralErr } From d0ebd6a83fabab7cdabfdf598866235f012e0e5d Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 6 Nov 2023 21:59:10 +0000 Subject: [PATCH 32/42] better handling for dangling watches - always replace the watch set so we don't end up in a bad state Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 58 +++++++++++++++++++------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 89d426c6e01..890cfc53c48 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -130,7 +130,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // in the manageCache loop. err := c.replaceWatchSet(ctx) - if general, failedGVKs, _ := interpretErr(err, newGVKs); len(failedGVKs) > 0 || general { + if general, failedGVKs, danglingWatches := interpretErr(err, newGVKs); len(failedGVKs) > 0 || general { var gvksToTryCancel []schema.GroupVersionKind if general { // if the err is general, assume all gvks need TryCancel because of some @@ -138,6 +138,8 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke gvksToTryCancel = c.gvksToSync.GVKs() } else { gvksToTryCancel = failedGVKs + + c.handleDanglingWatches(danglingWatches) } for _, g := range gvksToTryCancel { @@ -157,20 +159,16 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet.Add(c.gvksToSync.GVKs()...) c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) - if c.danglingWatches || !c.watchedSet.Equals(newWatchSet) { - var err error - c.watchedSet.Replace(newWatchSet, func() { - // *Note the following steps are not transactional with respect to admission control - - // Important: dynamic watches update must happen *after* updating our watchSet. - // Otherwise, the sync controller will drop events for the newly watched kinds. - err = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) - }) + var err error + c.watchedSet.Replace(newWatchSet, func() { + // *Note the following steps are not transactional with respect to admission control - return err - } + // Important: dynamic watches update must happen *after* updating our watchSet. + // Otherwise, the sync controller will drop events for the newly watched kinds. + err = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) + }) - return nil + return err } // interpret if the err received is general or whether it is specific to the provided GVKs. @@ -207,11 +205,29 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke defer c.mu.Unlock() c.gvksToSync.Remove(sourceKey) - // watch removal will happen async in manageCache() + err := c.replaceWatchSet(ctx) + general, _, danglingWatches := interpretErr(err, []schema.GroupVersionKind{}) + if general { + return fmt.Errorf("error establishing watches: %w", err) + } + + // watch removal retries for any dangling watches will happen async in manageCache() + c.handleDanglingWatches(danglingWatches) return nil } +// Mark if there are any dangling watches that need to be retried to be removed. +// assumes caller has lock. +func (c *CacheManager) handleDanglingWatches(danglingWatches bool) { + // if there were dangling watches previously but not anymore, + // mark the data cache as needing a wipe so we don't track unwanted resources. + if c.danglingWatches && !danglingWatches { + c.forceWipe = true + } + c.danglingWatches = danglingWatches +} + // ExcludeProcesses swaps the current process excluder with the new *process.Excluder. // It's a no-op if the two excluders are equal. func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { @@ -375,19 +391,13 @@ func (c *CacheManager) manageCache(ctx context.Context) { defer c.mu.Unlock() // first make sure there is no drift between c.gvksToSync and watch manager - err := c.replaceWatchSet(ctx) - _, _, danglingWatches := interpretErr(err, []schema.GroupVersionKind{}) - - // if there were dangling watches previously but not anymore, - // force a cache wipe so we don't track unwanted resources. - if c.danglingWatches && !danglingWatches { - c.forceWipe = true + if c.danglingWatches { + err := c.replaceWatchSet(ctx) + _, _, danglingWatches := interpretErr(err, []schema.GroupVersionKind{}) + c.handleDanglingWatches(danglingWatches) } - c.danglingWatches = danglingWatches - // perform a wipe if we removed gvks or the excluder changed c.wipeCacheIfNeeded(ctx) - if !c.needToList { // this means that there are no changes needed // such that any gvks need to be relisted. From eeba2bf85f06ac005654ebf582375394625c1987 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 7 Nov 2023 02:06:25 +0000 Subject: [PATCH 33/42] check err nullability, naming Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 12 ++++++++---- pkg/watch/errorlist.go | 6 +++--- pkg/watch/errorlist_test.go | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 890cfc53c48..7a90f46f222 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -168,7 +168,11 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { err = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) }) - return err + if err != nil { + return err + } + + return nil } // interpret if the err received is general or whether it is specific to the provided GVKs. @@ -184,19 +188,19 @@ func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.Group } failedGvksToAdd := watch.NewSet() - failedGvksToAdd.Add(f.FailingGVKsToAdd()...) + failedGvksToAdd.Add(f.AddGVKFailures()...) sourceGVKSet := watch.NewSet() sourceGVKSet.Add(gvks...) common := failedGvksToAdd.Intersection(sourceGVKSet) if common.Size() > 0 { - return false, common.Items(), f.HasFailingGVKsToRemove() + return false, common.Items(), f.HasRemoveGVKErrors() } // this error is not about the gvks in this request // but we still log it for visibility log.V(logging.DebugLevel).Info("encountered unrelated error when replacing watch set", "error", e) - return false, nil, f.HasFailingGVKsToRemove() + return false, nil, f.HasRemoveGVKErrors() } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index 62e86202634..5a90ca57353 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -65,7 +65,7 @@ func (e *ErrorList) Error() string { } // Return gvks for which there were errors adding watches. -func (e *ErrorList) FailingGVKsToAdd() []schema.GroupVersionKind { +func (e *ErrorList) AddGVKFailures() []schema.GroupVersionKind { gvks := []schema.GroupVersionKind{} for _, err := range e.errs { var gvkErr gvkErr @@ -77,8 +77,8 @@ func (e *ErrorList) FailingGVKsToAdd() []schema.GroupVersionKind { return gvks } -// Return gvks for which there were errors removing watches. -func (e *ErrorList) HasFailingGVKsToRemove() bool { +// Return whether any of the errors are because of a watch failing to be removed. +func (e *ErrorList) HasRemoveGVKErrors() bool { for _, err := range e.errs { var gvkErr gvkErr if errors.As(err, &gvkErr) && gvkErr.isRemove { diff --git a/pkg/watch/errorlist_test.go b/pkg/watch/errorlist_test.go index 555cc1c5ccf..3e3e0f12f93 100644 --- a/pkg/watch/errorlist_test.go +++ b/pkg/watch/errorlist_test.go @@ -72,7 +72,7 @@ func Test_WatchesError(t *testing.T) { er.Err(err) } - require.ElementsMatch(t, tc.expectedGVKs, er.FailingGVKsToAdd()) + require.ElementsMatch(t, tc.expectedGVKs, er.AddGVKFailures()) require.Equal(t, tc.generalErr, er.HasGeneralErr()) }) } From 034e8263a0cb82ff6720209a519c3b27b98d8bd5 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 7 Nov 2023 22:00:53 +0000 Subject: [PATCH 34/42] use remove set instead of forceWipe Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 61 ++++++++++++++------------- pkg/cachemanager/cachemanager_test.go | 37 ++++++++-------- pkg/watch/errorlist.go | 9 ++-- 3 files changed, 53 insertions(+), 54 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 7a90f46f222..ee5509bc68e 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -53,7 +53,6 @@ type CacheManager struct { gvksToDeleteFromCache *watch.Set excluderChanged bool danglingWatches bool - forceWipe bool // mu guards access to any of the fields above mu sync.RWMutex @@ -130,22 +129,23 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // in the manageCache loop. err := c.replaceWatchSet(ctx) - if general, failedGVKs, danglingWatches := interpretErr(err, newGVKs); len(failedGVKs) > 0 || general { - var gvksToTryCancel []schema.GroupVersionKind - if general { - // if the err is general, assume all gvks need TryCancel because of some - // WatchManager internal error and we don't want to block readiness. - gvksToTryCancel = c.gvksToSync.GVKs() - } else { - gvksToTryCancel = failedGVKs - - c.handleDanglingWatches(danglingWatches) - } + general, addGVKFailures, removeGVKFailures := interpretErr(err, newGVKs) + var gvksToTryCancel []schema.GroupVersionKind + if general { + // if the err is general, assume all gvks need TryCancel because of some + // WatchManager internal error and we don't want to block readiness. + gvksToTryCancel = c.gvksToSync.GVKs() + } else { + gvksToTryCancel = addGVKFailures - for _, g := range gvksToTryCancel { - c.tracker.TryCancelData(g) - } + c.handleDanglingWatches(removeGVKFailures) + } + + for _, g := range gvksToTryCancel { + c.tracker.TryCancelData(g) + } + if len(addGVKFailures) > 0 || general { return fmt.Errorf("error establishing watches: %w", err) } @@ -175,16 +175,16 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return nil } -// interpret if the err received is general or whether it is specific to the provided GVKs. -// also returns whether there were any errors removing watches. -func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind, bool) { +// interpret if the err received is general. If it is not, returns any GVKs for which we failed to add watches +// and are in common with the given gvks parameter. Also returns GVKs for which we failed to remove watches. +func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind, []schema.GroupVersionKind) { if e == nil { - return false, nil, false + return false, nil, nil } f := watch.NewErrorList() if !errors.As(e, &f) || f.HasGeneralErr() { - return true, nil, false + return true, nil, nil } failedGvksToAdd := watch.NewSet() @@ -194,13 +194,13 @@ func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.Group common := failedGvksToAdd.Intersection(sourceGVKSet) if common.Size() > 0 { - return false, common.Items(), f.HasRemoveGVKErrors() + return false, common.Items(), f.RemoveGVKFailures() } // this error is not about the gvks in this request // but we still log it for visibility log.V(logging.DebugLevel).Info("encountered unrelated error when replacing watch set", "error", e) - return false, nil, f.HasRemoveGVKErrors() + return false, nil, f.RemoveGVKFailures() } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. @@ -210,24 +210,26 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke c.gvksToSync.Remove(sourceKey) err := c.replaceWatchSet(ctx) - general, _, danglingWatches := interpretErr(err, []schema.GroupVersionKind{}) + general, _, removeGVKFailures := interpretErr(err, []schema.GroupVersionKind{}) if general { return fmt.Errorf("error establishing watches: %w", err) } // watch removal retries for any dangling watches will happen async in manageCache() - c.handleDanglingWatches(danglingWatches) + c.handleDanglingWatches(removeGVKFailures) return nil } // Mark if there are any dangling watches that need to be retried to be removed. // assumes caller has lock. -func (c *CacheManager) handleDanglingWatches(danglingWatches bool) { +func (c *CacheManager) handleDanglingWatches(removeGVKFailures []schema.GroupVersionKind) { + danglingWatches := len(removeGVKFailures) > 0 + // if there were dangling watches previously but not anymore, // mark the data cache as needing a wipe so we don't track unwanted resources. if c.danglingWatches && !danglingWatches { - c.forceWipe = true + c.gvksToDeleteFromCache.Add(removeGVKFailures...) } c.danglingWatches = danglingWatches } @@ -397,8 +399,8 @@ func (c *CacheManager) manageCache(ctx context.Context) { // first make sure there is no drift between c.gvksToSync and watch manager if c.danglingWatches { err := c.replaceWatchSet(ctx) - _, _, danglingWatches := interpretErr(err, []schema.GroupVersionKind{}) - c.handleDanglingWatches(danglingWatches) + _, _, removeGVKFailures := interpretErr(err, []schema.GroupVersionKind{}) + c.handleDanglingWatches(removeGVKFailures) } c.wipeCacheIfNeeded(ctx) @@ -493,14 +495,13 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro func (c *CacheManager) wipeCacheIfNeeded(ctx context.Context) { // remove any gvks not needing to be synced anymore // or re evaluate all if the excluder changed. - if c.gvksToDeleteFromCache.Size() > 0 || c.excluderChanged || c.forceWipe { + if c.gvksToDeleteFromCache.Size() > 0 || c.excluderChanged { if err := c.wipeData(ctx); err != nil { log.Error(err, "internal: error wiping cache") return } c.gvksToDeleteFromCache = watch.NewSet() - c.forceWipe = false c.excluderChanged = false c.needToList = true } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 151816827c9..1620b31d52c 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -628,22 +628,21 @@ func Test_interpretErr(t *testing.T) { gvk2Err.RemoveGVKErr(gvk2, someErr) cases := []struct { - name string - inputErr error - inputGVK []schema.GroupVersionKind - expectedFailingGVKs []schema.GroupVersionKind - expectGeneral bool - expectDanglingWatches bool + name string + inputErr error + inputGVK []schema.GroupVersionKind + expectedAddGVKFailures []schema.GroupVersionKind + expectedRemoveGVKFailures []schema.GroupVersionKind + expectGeneral bool }{ { - name: "nil err", - inputErr: nil, + name: "nil err", }, { - name: "intersection exists", - inputErr: fmt.Errorf("some err: %w", gvk1Err), - inputGVK: []schema.GroupVersionKind{gvk1}, - expectedFailingGVKs: []schema.GroupVersionKind{gvk1}, + name: "intersection exists", + inputErr: fmt.Errorf("some err: %w", gvk1Err), + inputGVK: []schema.GroupVersionKind{gvk1}, + expectedAddGVKFailures: []schema.GroupVersionKind{gvk1}, }, { name: "intersection does not exist", @@ -651,11 +650,9 @@ func Test_interpretErr(t *testing.T) { inputGVK: []schema.GroupVersionKind{gvk2}, }, { - name: "intersection exists but we are removing the gvk", - inputErr: gvk2Err, - inputGVK: []schema.GroupVersionKind{gvk2}, - expectedFailingGVKs: []schema.GroupVersionKind{}, - expectDanglingWatches: true, + name: "gvk watch failing to remove", + inputErr: gvk2Err, + expectedRemoveGVKFailures: []schema.GroupVersionKind{gvk2}, }, { name: "non-watchmanager error reports general error with no GVKs", @@ -673,11 +670,11 @@ func Test_interpretErr(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - general, gvks, danglingWatches := interpretErr(tc.inputErr, tc.inputGVK) + general, addGVKs, removeGVKs := interpretErr(tc.inputErr, tc.inputGVK) require.Equal(t, tc.expectGeneral, general) - require.ElementsMatch(t, gvks, tc.expectedFailingGVKs) - require.Equal(t, tc.expectDanglingWatches, danglingWatches) + require.ElementsMatch(t, addGVKs, tc.expectedAddGVKFailures) + require.ElementsMatch(t, removeGVKs, tc.expectedRemoveGVKFailures) }) } } diff --git a/pkg/watch/errorlist.go b/pkg/watch/errorlist.go index 5a90ca57353..8df19edcf76 100644 --- a/pkg/watch/errorlist.go +++ b/pkg/watch/errorlist.go @@ -77,16 +77,17 @@ func (e *ErrorList) AddGVKFailures() []schema.GroupVersionKind { return gvks } -// Return whether any of the errors are because of a watch failing to be removed. -func (e *ErrorList) HasRemoveGVKErrors() bool { +// Return gvks for which there were errors removing watches. +func (e *ErrorList) RemoveGVKFailures() []schema.GroupVersionKind { + gvks := []schema.GroupVersionKind{} for _, err := range e.errs { var gvkErr gvkErr if errors.As(err, &gvkErr) && gvkErr.isRemove { - return true + gvks = append(gvks, gvkErr.gvk) } } - return false + return gvks } func (e *ErrorList) HasGeneralErr() bool { From 854306ea499b23a503526f78c079ba0bca9c926f Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 9 Nov 2023 19:14:23 +0000 Subject: [PATCH 35/42] set instead of book for dangling watches Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 51 +++++++++--------- pkg/cachemanager/cachemanager_test.go | 75 +++++++++++++++++++++++---- pkg/watch/set.go | 8 +++ 3 files changed, 96 insertions(+), 38 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index ee5509bc68e..bbacac1e05c 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -51,8 +51,8 @@ type CacheManager struct { gvksToSync *aggregator.GVKAgreggator needToList bool gvksToDeleteFromCache *watch.Set + danglingWatches *watch.Set // gvks whose watches have failed to be removed excluderChanged bool - danglingWatches bool // mu guards access to any of the fields above mu sync.RWMutex @@ -100,6 +100,7 @@ func NewCacheManager(config *Config) (*CacheManager, error) { gvksToSync: config.GVKAggregator, backgroundManagementTicker: *time.NewTicker(3 * time.Second), gvksToDeleteFromCache: watch.NewSet(), + danglingWatches: watch.NewSet(), } return cm, nil @@ -129,7 +130,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // in the manageCache loop. err := c.replaceWatchSet(ctx) - general, addGVKFailures, removeGVKFailures := interpretErr(err, newGVKs) + general, addGVKFailures := interpretErr(err, newGVKs) var gvksToTryCancel []schema.GroupVersionKind if general { // if the err is general, assume all gvks need TryCancel because of some @@ -137,8 +138,6 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke gvksToTryCancel = c.gvksToSync.GVKs() } else { gvksToTryCancel = addGVKFailures - - c.handleDanglingWatches(removeGVKFailures) } for _, g := range gvksToTryCancel { @@ -169,22 +168,25 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { }) if err != nil { + // account for any watches failing to remove + if f := watch.NewErrorList(); errors.As(err, &f) { + c.handleDanglingWatches(f.RemoveGVKFailures()) + } return err } return nil } -// interpret if the err received is general. If it is not, returns any GVKs for which we failed to add watches -// and are in common with the given gvks parameter. Also returns GVKs for which we failed to remove watches. -func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind, []schema.GroupVersionKind) { +// interpret if the err received is general or whether it is specific to the provided GVKs. +func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind) { if e == nil { - return false, nil, nil + return false, nil } f := watch.NewErrorList() if !errors.As(e, &f) || f.HasGeneralErr() { - return true, nil, nil + return true, nil } failedGvksToAdd := watch.NewSet() @@ -194,13 +196,13 @@ func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.Group common := failedGvksToAdd.Intersection(sourceGVKSet) if common.Size() > 0 { - return false, common.Items(), f.RemoveGVKFailures() + return false, common.Items() } // this error is not about the gvks in this request // but we still log it for visibility log.V(logging.DebugLevel).Info("encountered unrelated error when replacing watch set", "error", e) - return false, nil, f.RemoveGVKFailures() + return false, nil } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. @@ -210,28 +212,24 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke c.gvksToSync.Remove(sourceKey) err := c.replaceWatchSet(ctx) - general, _, removeGVKFailures := interpretErr(err, []schema.GroupVersionKind{}) - if general { + if general, _ := interpretErr(err, []schema.GroupVersionKind{}); general { return fmt.Errorf("error establishing watches: %w", err) } - // watch removal retries for any dangling watches will happen async in manageCache() - c.handleDanglingWatches(removeGVKFailures) - return nil } -// Mark if there are any dangling watches that need to be retried to be removed. +// Mark if there are any dangling watches that need to be retried to be removed and which +// of the watches have finally been removed so they can be wiped from the cache. // assumes caller has lock. func (c *CacheManager) handleDanglingWatches(removeGVKFailures []schema.GroupVersionKind) { - danglingWatches := len(removeGVKFailures) > 0 + removeGVKFailuresSet := watch.SetFrom(removeGVKFailures) - // if there were dangling watches previously but not anymore, - // mark the data cache as needing a wipe so we don't track unwanted resources. - if c.danglingWatches && !danglingWatches { - c.gvksToDeleteFromCache.Add(removeGVKFailures...) - } - c.danglingWatches = danglingWatches + // any watches that failed previously but not anymore need to be marked for deletion + finallyRemoved := c.danglingWatches.Difference(removeGVKFailuresSet) + c.gvksToDeleteFromCache.AddSet(finallyRemoved) + + c.danglingWatches.RemoveSet(finallyRemoved) } // ExcludeProcesses swaps the current process excluder with the new *process.Excluder. @@ -397,10 +395,9 @@ func (c *CacheManager) manageCache(ctx context.Context) { defer c.mu.Unlock() // first make sure there is no drift between c.gvksToSync and watch manager - if c.danglingWatches { + if c.danglingWatches.Size() > 0 { err := c.replaceWatchSet(ctx) - _, _, removeGVKFailures := interpretErr(err, []schema.GroupVersionKind{}) - c.handleDanglingWatches(removeGVKFailures) + log.V(logging.DebugLevel).Info("error replacing watch set", "error", err) } c.wipeCacheIfNeeded(ctx) diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 1620b31d52c..397cc543196 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -628,12 +628,11 @@ func Test_interpretErr(t *testing.T) { gvk2Err.RemoveGVKErr(gvk2, someErr) cases := []struct { - name string - inputErr error - inputGVK []schema.GroupVersionKind - expectedAddGVKFailures []schema.GroupVersionKind - expectedRemoveGVKFailures []schema.GroupVersionKind - expectGeneral bool + name string + inputErr error + inputGVK []schema.GroupVersionKind + expectedAddGVKFailures []schema.GroupVersionKind + expectGeneral bool }{ { name: "nil err", @@ -650,9 +649,8 @@ func Test_interpretErr(t *testing.T) { inputGVK: []schema.GroupVersionKind{gvk2}, }, { - name: "gvk watch failing to remove", - inputErr: gvk2Err, - expectedRemoveGVKFailures: []schema.GroupVersionKind{gvk2}, + name: "gvk watch failing to remove", + inputErr: gvk2Err, }, { name: "non-watchmanager error reports general error with no GVKs", @@ -670,11 +668,66 @@ func Test_interpretErr(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - general, addGVKs, removeGVKs := interpretErr(tc.inputErr, tc.inputGVK) + general, addGVKs := interpretErr(tc.inputErr, tc.inputGVK) require.Equal(t, tc.expectGeneral, general) require.ElementsMatch(t, addGVKs, tc.expectedAddGVKFailures) - require.ElementsMatch(t, removeGVKs, tc.expectedRemoveGVKFailures) + }) + } +} + +func Test_handleDanglingWatches(t *testing.T) { + gvk1 := schema.GroupVersionKind{Group: "g1", Version: "v1", Kind: "k1"} + gvk2 := schema.GroupVersionKind{Group: "g2", Version: "v2", Kind: "k2"} + + cases := []struct { + name string + alreadyDangling *watch.Set + removeGVKFailures []schema.GroupVersionKind + expectedDangling *watch.Set + }{ + { + name: "no watches dangling, nothing to remove", + expectedDangling: watch.NewSet(), + }, + { + name: "no watches dangling, something to remove", + expectedDangling: watch.NewSet(), + }, + { + name: "watches dangling, finally removed", + alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + removeGVKFailures: []schema.GroupVersionKind{}, + expectedDangling: watch.SetFrom([]schema.GroupVersionKind{}), + }, + { + name: "watches dangling, keep dangling", + alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + removeGVKFailures: []schema.GroupVersionKind{gvk1}, + expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + }, + { + name: "watches dangling, some keep dangling", + alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk2, gvk1}), + removeGVKFailures: []schema.GroupVersionKind{gvk1}, + expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cm, _ := makeCacheManager(t) + if tc.alreadyDangling != nil { + cm.danglingWatches.AddSet(tc.alreadyDangling) + } + + cm.handleDanglingWatches(tc.removeGVKFailures) + + if tc.expectedDangling != nil { + require.ElementsMatch(t, tc.expectedDangling.Items(), cm.danglingWatches.Items()) + } else { + require.Empty(t, cm.danglingWatches) + } }) } } diff --git a/pkg/watch/set.go b/pkg/watch/set.go index 30e1b0cfae4..5d48d576605 100644 --- a/pkg/watch/set.go +++ b/pkg/watch/set.go @@ -64,6 +64,14 @@ func NewSet() *Set { } } +// SetFrom constructs a new watchSet from the given gvks. +func SetFrom(items []schema.GroupVersionKind) *Set { + s := NewSet() + s.Add(items...) + + return s +} + func (w *Set) Size() int { w.mux.RLock() defer w.mux.RUnlock() From d7bb9b5b1a1ce05f500de3031bf7a6beed5dbc1a Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 9 Nov 2023 23:07:05 +0000 Subject: [PATCH 36/42] general error watch dangling Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 25 +++++++++++++++---------- pkg/cachemanager/cachemanager_test.go | 26 ++++++++++++-------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index bbacac1e05c..9282d8bed71 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -154,9 +154,10 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke // replaceWatchSet looks at the gvksToSync and makes changes to the registrar's watch set. // Assumes caller has lock. On error, actual watch state may not align with intended watch state. func (c *CacheManager) replaceWatchSet(ctx context.Context) error { - newWatchSet := watch.NewSet() - newWatchSet.Add(c.gvksToSync.GVKs()...) - c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) + newWatchSet := watch.SetFrom(c.gvksToSync.GVKs()) + + gvksToRemove := c.watchedSet.Difference(newWatchSet) + c.gvksToDeleteFromCache.AddSet(gvksToRemove) var err error c.watchedSet.Replace(newWatchSet, func() { @@ -169,9 +170,13 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { if err != nil { // account for any watches failing to remove - if f := watch.NewErrorList(); errors.As(err, &f) { - c.handleDanglingWatches(f.RemoveGVKFailures()) + if f := watch.NewErrorList(); errors.As(err, &f) && !f.HasGeneralErr() { + c.handleDanglingWatches(watch.SetFrom(f.RemoveGVKFailures())) + } else { + // defensively assume all watches that needed removal failed to be removed in the general error case + c.handleDanglingWatches(gvksToRemove) } + return err } @@ -222,14 +227,14 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke // Mark if there are any dangling watches that need to be retried to be removed and which // of the watches have finally been removed so they can be wiped from the cache. // assumes caller has lock. -func (c *CacheManager) handleDanglingWatches(removeGVKFailures []schema.GroupVersionKind) { - removeGVKFailuresSet := watch.SetFrom(removeGVKFailures) - +func (c *CacheManager) handleDanglingWatches(removeGVKFailures *watch.Set) { // any watches that failed previously but not anymore need to be marked for deletion - finallyRemoved := c.danglingWatches.Difference(removeGVKFailuresSet) + finallyRemoved := c.danglingWatches.Difference(removeGVKFailures) c.gvksToDeleteFromCache.AddSet(finallyRemoved) - c.danglingWatches.RemoveSet(finallyRemoved) + + // keep track of new dangling watches + c.danglingWatches.AddSet(removeGVKFailures.Difference(c.danglingWatches)) } // ExcludeProcesses swaps the current process excluder with the new *process.Excluder. diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 397cc543196..f0287526d29 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -683,33 +683,35 @@ func Test_handleDanglingWatches(t *testing.T) { cases := []struct { name string alreadyDangling *watch.Set - removeGVKFailures []schema.GroupVersionKind + removeGVKFailures *watch.Set expectedDangling *watch.Set }{ { - name: "no watches dangling, nothing to remove", - expectedDangling: watch.NewSet(), + name: "no watches dangling, nothing to remove", + removeGVKFailures: watch.NewSet(), + expectedDangling: watch.NewSet(), }, { - name: "no watches dangling, something to remove", - expectedDangling: watch.NewSet(), + name: "no watches dangling, something to remove", + removeGVKFailures: watch.NewSet(), + expectedDangling: watch.NewSet(), }, { name: "watches dangling, finally removed", alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - removeGVKFailures: []schema.GroupVersionKind{}, - expectedDangling: watch.SetFrom([]schema.GroupVersionKind{}), + removeGVKFailures: watch.NewSet(), + expectedDangling: watch.NewSet(), }, { name: "watches dangling, keep dangling", alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - removeGVKFailures: []schema.GroupVersionKind{gvk1}, + removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), }, { name: "watches dangling, some keep dangling", alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk2, gvk1}), - removeGVKFailures: []schema.GroupVersionKind{gvk1}, + removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), }, } @@ -723,11 +725,7 @@ func Test_handleDanglingWatches(t *testing.T) { cm.handleDanglingWatches(tc.removeGVKFailures) - if tc.expectedDangling != nil { - require.ElementsMatch(t, tc.expectedDangling.Items(), cm.danglingWatches.Items()) - } else { - require.Empty(t, cm.danglingWatches) - } + require.ElementsMatch(t, tc.expectedDangling.Items(), cm.danglingWatches.Items()) }) } } From 7faecbca785ce718203db5fc652324a52a1eb897 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 10 Nov 2023 22:40:01 +0000 Subject: [PATCH 37/42] review feedback Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/aggregator/aggregator.go | 3 +- pkg/cachemanager/cachemanager.go | 5 +- pkg/cachemanager/cachemanager_test.go | 51 +++++++++++-------- .../syncset/syncset_controller_test.go | 20 ++++---- pkg/watch/errorlist_test.go | 49 +++++++++++++----- 5 files changed, 81 insertions(+), 47 deletions(-) diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index d573f368881..0b8785e02f4 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -122,8 +122,7 @@ func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struc for gvk := range gvks { keySet, found := b.reverseStore[gvk] if !found { - // this should not happen if we keep the two maps well defined - // but let's be defensive nonetheless. + // by definition, nothing to prune return } diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 9282d8bed71..246fa11b502 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -183,7 +183,8 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return nil } -// interpret if the err received is general or whether it is specific to the provided GVKs. +// interpretErr determines if the passed-in error is general (not GVK-specific) and, +// if GVK-specific, returns the subset of the passed in GVKs that are included in the err. func interpretErr(e error, gvks []schema.GroupVersionKind) (bool, []schema.GroupVersionKind) { if e == nil { return false, nil @@ -217,6 +218,8 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke c.gvksToSync.Remove(sourceKey) err := c.replaceWatchSet(ctx) + // Retrying watch deletion due to per-GVK errors is done in the background management loop, + // and thus only a general error should be returned to the caller for a controller-based retry. if general, _ := interpretErr(err, []schema.GroupVersionKind{}); general { return fmt.Errorf("error establishing watches: %w", err) } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index f0287526d29..cf1d626bbda 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -681,38 +681,44 @@ func Test_handleDanglingWatches(t *testing.T) { gvk2 := schema.GroupVersionKind{Group: "g2", Version: "v2", Kind: "k2"} cases := []struct { - name string - alreadyDangling *watch.Set - removeGVKFailures *watch.Set - expectedDangling *watch.Set + name string + alreadyDangling *watch.Set + removeGVKFailures *watch.Set + expectedDangling *watch.Set + expectedGVKsToDelete *watch.Set }{ { - name: "no watches dangling, nothing to remove", - removeGVKFailures: watch.NewSet(), - expectedDangling: watch.NewSet(), + name: "mothing dangling, no failures yields nothing dangling", + removeGVKFailures: watch.NewSet(), + expectedDangling: watch.NewSet(), + expectedGVKsToDelete: watch.NewSet(), }, { - name: "no watches dangling, something to remove", - removeGVKFailures: watch.NewSet(), - expectedDangling: watch.NewSet(), + name: "mothing dangling, per gvk failures yields something dangling", + removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + expectedGVKsToDelete: watch.NewSet(), }, { - name: "watches dangling, finally removed", - alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - removeGVKFailures: watch.NewSet(), - expectedDangling: watch.NewSet(), + name: "watches dangling, finally removed", + alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + removeGVKFailures: watch.NewSet(), + expectedDangling: watch.NewSet(), + expectedGVKsToDelete: watch.SetFrom([]schema.GroupVersionKind{gvk1}), }, { - name: "watches dangling, keep dangling", - alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + name: "watches dangling, keep dangling", + alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + expectedGVKsToDelete: watch.NewSet(), }, { - name: "watches dangling, some keep dangling", - alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk2, gvk1}), - removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + name: "watches dangling, some keep dangling", + alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk2, gvk1}), + removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), + expectedGVKsToDelete: watch.SetFrom([]schema.GroupVersionKind{gvk2}), }, } @@ -726,6 +732,7 @@ func Test_handleDanglingWatches(t *testing.T) { cm.handleDanglingWatches(tc.removeGVKFailures) require.ElementsMatch(t, tc.expectedDangling.Items(), cm.danglingWatches.Items()) + require.ElementsMatch(t, tc.expectedGVKsToDelete.Items(), cm.gvksToDeleteFromCache.Items()) }) } } diff --git a/pkg/controller/syncset/syncset_controller_test.go b/pkg/controller/syncset/syncset_controller_test.go index cd39a00c72c..7a05b9762a1 100644 --- a/pkg/controller/syncset/syncset_controller_test.go +++ b/pkg/controller/syncset/syncset_controller_test.go @@ -63,14 +63,14 @@ func Test_ReconcileSyncSet(t *testing.T) { expectedGVKs []schema.GroupVersionKind }{ { - name: "basic reconcile", + name: "SyncSet includes new GVKs", syncSources: []*syncsetv1alpha1.SyncSet{ fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{configMapGVK, nsGVK}), }, expectedGVKs: []schema.GroupVersionKind{configMapGVK, nsGVK}, }, { - name: "syncset adjusts spec", + name: "New SyncSet generation has one less GVK than previous generation", syncSources: []*syncsetv1alpha1.SyncSet{ fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{configMapGVK, nsGVK}), fakes.SyncSetFor("syncset1", []schema.GroupVersionKind{nsGVK}), @@ -82,20 +82,20 @@ func Test_ReconcileSyncSet(t *testing.T) { for _, tt := range tts { t.Run(tt.name, func(t *testing.T) { created := map[string]struct{}{} - for _, o := range tt.syncSources { - if _, ok := created[o.GetName()]; ok { - curObj, ok := o.DeepCopyObject().(client.Object) + for _, syncSource := range tt.syncSources { + if _, ok := created[syncSource.GetName()]; ok { + curObj, ok := syncSource.DeepCopyObject().(client.Object) require.True(t, ok) // eventually we should find the object require.Eventually(t, func() bool { return testRes.k8sclient.Get(ctx, client.ObjectKeyFromObject(curObj), curObj) == nil - }, timeout, tick, fmt.Sprintf("getting %s", o.GetName())) + }, timeout, tick, fmt.Sprintf("getting %s", syncSource.GetName())) - o.SetResourceVersion(curObj.GetResourceVersion()) - require.NoError(t, testRes.k8sclient.Update(ctx, o), fmt.Sprintf("updating %s", o.GetName())) + syncSource.SetResourceVersion(curObj.GetResourceVersion()) + require.NoError(t, testRes.k8sclient.Update(ctx, syncSource), fmt.Sprintf("updating %s", syncSource.GetName())) } else { - require.NoError(t, testRes.k8sclient.Create(ctx, o), fmt.Sprintf("creating %s", o.GetName())) - created[o.GetName()] = struct{}{} + require.NoError(t, testRes.k8sclient.Create(ctx, syncSource), fmt.Sprintf("creating %s", syncSource.GetName())) + created[syncSource.GetName()] = struct{}{} } } diff --git a/pkg/watch/errorlist_test.go b/pkg/watch/errorlist_test.go index 3e3e0f12f93..b096e1b9d96 100644 --- a/pkg/watch/errorlist_test.go +++ b/pkg/watch/errorlist_test.go @@ -13,6 +13,7 @@ func Test_WatchesError(t *testing.T) { someGVKA := schema.GroupVersionKind{Group: "a", Version: "b", Kind: "c"} someGVKB := schema.GroupVersionKind{Group: "x", Version: "y", Kind: "z"} someGVKC := schema.GroupVersionKind{Group: "m", Version: "n", Kind: "o"} + someGVKD := schema.GroupVersionKind{Group: "p", Version: "q", Kind: "r"} type errsToAdd struct { errs []error @@ -20,10 +21,11 @@ func Test_WatchesError(t *testing.T) { } tcs := []struct { - name string - errsToAdd errsToAdd - expectedGVKs []schema.GroupVersionKind - generalErr bool + name string + errsToAdd errsToAdd + expectedAddGVKs []schema.GroupVersionKind + expectedRemoveGVKs []schema.GroupVersionKind + expectGeneralErr bool }{ { name: "gvk errors, not general", @@ -31,10 +33,11 @@ func Test_WatchesError(t *testing.T) { gvkErrs: []gvkErr{ {err: someErr, gvk: someGVKA}, {err: someErr, gvk: someGVKB}, + {err: someErr, gvk: someGVKD, isRemove: true}, }, }, - expectedGVKs: []schema.GroupVersionKind{someGVKA, someGVKB}, - generalErr: false, + expectedAddGVKs: []schema.GroupVersionKind{someGVKA, someGVKB}, + expectedRemoveGVKs: []schema.GroupVersionKind{someGVKD}, }, { name: "gvk errors and general error", @@ -42,19 +45,40 @@ func Test_WatchesError(t *testing.T) { gvkErrs: []gvkErr{ {err: someErr, gvk: someGVKA}, {err: someErr, gvk: someGVKB}, - {err: someErr, gvk: someGVKC, isRemove: true}, // this one should not show up in FailingGVKsToAdd + {err: someErr, gvk: someGVKC, isRemove: true}, }, errs: []error{someErr, someErr}, }, - expectedGVKs: []schema.GroupVersionKind{someGVKA, someGVKB}, - generalErr: true, + expectedAddGVKs: []schema.GroupVersionKind{someGVKA, someGVKB}, + expectedRemoveGVKs: []schema.GroupVersionKind{someGVKC}, + expectGeneralErr: true, }, { name: "just general error", errsToAdd: errsToAdd{ errs: []error{someErr}, }, - generalErr: true, + expectGeneralErr: true, + }, + { + name: "just add gvk error", + errsToAdd: errsToAdd{ + gvkErrs: []gvkErr{ + {err: someErr, gvk: someGVKA}, + {err: someErr, gvk: someGVKB}, + }, + }, + expectedAddGVKs: []schema.GroupVersionKind{someGVKA, someGVKB}, + }, + { + name: "just remove gvk error", + errsToAdd: errsToAdd{ + gvkErrs: []gvkErr{ + {err: someErr, gvk: someGVKC, isRemove: true}, + {err: someErr, gvk: someGVKD, isRemove: true}, + }, + }, + expectedRemoveGVKs: []schema.GroupVersionKind{someGVKC, someGVKD}, }, } @@ -72,8 +96,9 @@ func Test_WatchesError(t *testing.T) { er.Err(err) } - require.ElementsMatch(t, tc.expectedGVKs, er.AddGVKFailures()) - require.Equal(t, tc.generalErr, er.HasGeneralErr()) + require.ElementsMatch(t, tc.expectedAddGVKs, er.AddGVKFailures()) + require.ElementsMatch(t, tc.expectedRemoveGVKs, er.RemoveGVKFailures()) + require.Equal(t, tc.expectGeneralErr, er.HasGeneralErr()) }) } } From 03565fa2e7e6fd5af280dda6c90ee38d4d23f13a Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 15 Nov 2023 23:53:58 +0000 Subject: [PATCH 38/42] refactor: use tt Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../aggregator/aggregator_test.go | 88 ++++++++++--------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/pkg/cachemanager/aggregator/aggregator_test.go b/pkg/cachemanager/aggregator/aggregator_test.go index 17a6e51f103..6aa9b76cac2 100644 --- a/pkg/cachemanager/aggregator/aggregator_test.go +++ b/pkg/cachemanager/aggregator/aggregator_test.go @@ -215,46 +215,54 @@ func Test_GVKAggregator_Upsert(t *testing.T) { } func Test_GVKAgreggator_Remove(t *testing.T) { - t.Run("Remove on empty aggregator", func(t *testing.T) { - b := NewGVKAggregator() - key := Key{Source: "testSource", ID: "testID"} - b.Remove(key) - }) - - t.Run("Remove non-existing key", func(t *testing.T) { - b := NewGVKAggregator() - key1 := Key{Source: syncset, ID: "testID1"} - key2 := Key{Source: configsync, ID: "testID2"} - gvks := []schema.GroupVersionKind{g1v1k1, g1v1k2} - b.Upsert(key1, gvks) - b.Remove(key2) - }) - - t.Run("Remove existing key and verify reverseStore", func(t *testing.T) { - b := NewGVKAggregator() - key1 := Key{Source: syncset, ID: "testID"} - gvks := []schema.GroupVersionKind{g1v1k1, g1v1k2} - b.Upsert(key1, gvks) - b.Remove(key1) - - for _, gvk := range gvks { - require.False(t, b.IsPresent(gvk)) - } - }) - - t.Run("Remove 1 of existing keys referencing GVKs", func(t *testing.T) { - b := NewGVKAggregator() - key1 := Key{Source: syncset, ID: "testID"} - key2 := Key{Source: configsync, ID: "testID"} - gvks := []schema.GroupVersionKind{g1v1k1, g1v1k2} - b.Upsert(key1, gvks) - b.Upsert(key2, gvks) - b.Remove(key1) - - for _, gvk := range gvks { - require.True(t, b.IsPresent(gvk)) - } - }) + tests := []struct { + name string + toUpsert []keyedGVKs + keyToRemove Key + expectedGVKs []schema.GroupVersionKind + }{ + { + name: "empty aggregator", + keyToRemove: Key{Source: "testSource", ID: "testID"}, + }, + { + name: "remove non existing key", + toUpsert: []keyedGVKs{ + {key: Key{Source: syncset, ID: "testID1"}, gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}}, + }, + keyToRemove: Key{Source: configsync, ID: "testID"}, + expectedGVKs: []schema.GroupVersionKind{g1v1k1, g1v1k2}, + }, + { + name: "remove existing key", + toUpsert: []keyedGVKs{ + {key: Key{Source: syncset, ID: "testID1"}, gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}}, + }, + keyToRemove: Key{Source: syncset, ID: "testID1"}, + }, + { + name: "remove overlapping key", + toUpsert: []keyedGVKs{ + {key: Key{Source: syncset, ID: "testID1"}, gvks: []schema.GroupVersionKind{g1v1k1, g1v1k2}}, + {key: Key{Source: configsync, ID: "testID2"}, gvks: []schema.GroupVersionKind{g1v1k2, g1v2k2}}, + }, + keyToRemove: Key{Source: syncset, ID: "testID1"}, + expectedGVKs: []schema.GroupVersionKind{g1v1k2, g1v2k2}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + agg := NewGVKAggregator() + for _, seed := range tc.toUpsert { + agg.Upsert(seed.key, seed.gvks) + } + + agg.Remove(tc.keyToRemove) + + require.ElementsMatch(t, tc.expectedGVKs, agg.GVKs()) + }) + } } // Test_GVKAggreggator_E2E is a test that: From 3766d66a479e0103b989e2103184e96febb372fb Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 15 Nov 2023 23:54:37 +0000 Subject: [PATCH 39/42] fix: general err dangling watches Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 33 ++++++------ pkg/cachemanager/cachemanager_test.go | 67 +++++++++++++++++++----- pkg/readiness/ready_tracker_unit_test.go | 4 +- 3 files changed, 74 insertions(+), 30 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 246fa11b502..42678933cb9 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -60,7 +60,7 @@ type CacheManager struct { cfClient CFDataClient syncMetricsCache *syncutil.MetricsCache tracker *readiness.Tracker - registrar *watch.Registrar + registrar registrarReplacer backgroundManagementTicker time.Ticker reader client.Reader } @@ -71,6 +71,10 @@ type CFDataClient interface { RemoveData(ctx context.Context, data interface{}) (*types.Responses, error) } +type registrarReplacer interface { + ReplaceWatch(ctx context.Context, gvks []schema.GroupVersionKind) error +} + func NewCacheManager(config *Config) (*CacheManager, error) { if config.Registrar == nil { return nil, fmt.Errorf("registrar must be non-nil") @@ -171,15 +175,25 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { if err != nil { // account for any watches failing to remove if f := watch.NewErrorList(); errors.As(err, &f) && !f.HasGeneralErr() { - c.handleDanglingWatches(watch.SetFrom(f.RemoveGVKFailures())) + removeGVKFailures := watch.SetFrom(f.RemoveGVKFailures()) + finallyRemoved := c.danglingWatches.Difference(removeGVKFailures) + + c.gvksToDeleteFromCache.AddSet(finallyRemoved) + c.danglingWatches.RemoveSet(finallyRemoved) + c.danglingWatches.AddSet(removeGVKFailures.Difference(c.danglingWatches)) } else { // defensively assume all watches that needed removal failed to be removed in the general error case - c.handleDanglingWatches(gvksToRemove) + // also assume whatever watches were dangling are still dangling. + c.danglingWatches.AddSet(gvksToRemove) } return err } + // if no error, it means no previously dangling watches are still dangling + c.gvksToDeleteFromCache.AddSet(c.danglingWatches) + c.danglingWatches = watch.NewSet() + return nil } @@ -227,19 +241,6 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke return nil } -// Mark if there are any dangling watches that need to be retried to be removed and which -// of the watches have finally been removed so they can be wiped from the cache. -// assumes caller has lock. -func (c *CacheManager) handleDanglingWatches(removeGVKFailures *watch.Set) { - // any watches that failed previously but not anymore need to be marked for deletion - finallyRemoved := c.danglingWatches.Difference(removeGVKFailures) - c.gvksToDeleteFromCache.AddSet(finallyRemoved) - c.danglingWatches.RemoveSet(finallyRemoved) - - // keep track of new dangling watches - c.danglingWatches.AddSet(removeGVKFailures.Difference(c.danglingWatches)) -} - // ExcludeProcesses swaps the current process excluder with the new *process.Excluder. // It's a no-op if the two excluders are equal. func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index cf1d626bbda..1628846bb64 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -676,28 +676,30 @@ func Test_interpretErr(t *testing.T) { } } -func Test_handleDanglingWatches(t *testing.T) { +func Test_replaceWatchSet_danglingWatches(t *testing.T) { gvk1 := schema.GroupVersionKind{Group: "g1", Version: "v1", Kind: "k1"} gvk2 := schema.GroupVersionKind{Group: "g2", Version: "v2", Kind: "k2"} cases := []struct { - name string - alreadyDangling *watch.Set - removeGVKFailures *watch.Set + name string + alreadyDangling *watch.Set + removeGVKFailures *watch.Set + hasGeneralErr bool + expectedDangling *watch.Set expectedGVKsToDelete *watch.Set }{ { - name: "mothing dangling, no failures yields nothing dangling", + name: "nothing dangling, no failures yields nothing dangling", removeGVKFailures: watch.NewSet(), expectedDangling: watch.NewSet(), expectedGVKsToDelete: watch.NewSet(), }, { - name: "mothing dangling, per gvk failures yields something dangling", + name: "nothing dangling, per gvk failures yields something dangling", removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - expectedGVKsToDelete: watch.NewSet(), + expectedGVKsToDelete: watch.SetFrom([]schema.GroupVersionKind{gvk1}), }, { name: "watches dangling, finally removed", @@ -711,28 +713,69 @@ func Test_handleDanglingWatches(t *testing.T) { alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - expectedGVKsToDelete: watch.NewSet(), + expectedGVKsToDelete: watch.SetFrom([]schema.GroupVersionKind{gvk1}), }, { name: "watches dangling, some keep dangling", - alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk2, gvk1}), + alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1, gvk2}), removeGVKFailures: watch.SetFrom([]schema.GroupVersionKind{gvk1}), expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1}), - expectedGVKsToDelete: watch.SetFrom([]schema.GroupVersionKind{gvk2}), + expectedGVKsToDelete: watch.SetFrom([]schema.GroupVersionKind{gvk1, gvk2}), + }, + { + name: "watches dangling, keep dangling on general error", + alreadyDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1, gvk2}), + removeGVKFailures: watch.NewSet(), + expectedDangling: watch.SetFrom([]schema.GroupVersionKind{gvk1, gvk2}), + expectedGVKsToDelete: watch.NewSet(), + hasGeneralErr: true, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - cm, _ := makeCacheManager(t) + cm, ctx := makeCacheManager(t) + fakeRegistrar := &fakeRegistrar{removeGVKFailures: tc.removeGVKFailures.Items()} + if tc.hasGeneralErr { + fakeRegistrar.generalErr = errors.New("test general error") + } + cm.registrar = fakeRegistrar + if tc.alreadyDangling != nil { cm.danglingWatches.AddSet(tc.alreadyDangling) } + cm.watchedSet.AddSet(tc.removeGVKFailures) - cm.handleDanglingWatches(tc.removeGVKFailures) + err := cm.replaceWatchSet(ctx) + if tc.removeGVKFailures.Size() == 0 && !tc.hasGeneralErr { + require.NoError(t, err) + } require.ElementsMatch(t, tc.expectedDangling.Items(), cm.danglingWatches.Items()) require.ElementsMatch(t, tc.expectedGVKsToDelete.Items(), cm.gvksToDeleteFromCache.Items()) }) } } + +type fakeRegistrar struct { + removeGVKFailures []schema.GroupVersionKind + generalErr error +} + +func (f *fakeRegistrar) ReplaceWatch(ctx context.Context, gvks []schema.GroupVersionKind) error { + err := watch.NewErrorList() + if f.generalErr != nil { + err.Err(f.generalErr) + } + if len(f.removeGVKFailures) > 0 { + for _, gvk := range f.removeGVKFailures { + err.RemoveGVKErr(gvk, errors.New("some error")) + } + } + + if err.Size() != 0 { + return err + } + + return nil +} diff --git a/pkg/readiness/ready_tracker_unit_test.go b/pkg/readiness/ready_tracker_unit_test.go index 11d28213036..8cc084440ad 100644 --- a/pkg/readiness/ready_tracker_unit_test.go +++ b/pkg/readiness/ready_tracker_unit_test.go @@ -77,7 +77,7 @@ func init() { // Verify that TryCancelTemplate functions the same as regular CancelTemplate if readinessRetries is set to 0. func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { - lister := fake.NewClientBuilder().WithRuntimeObjects(&convertedTemplate).Build() + lister := fake.NewClientBuilder().WithRuntimeObjects(convertedTemplate.DeepCopyObject()).Build() rt := newTracker(lister, false, false, false, func() objData { return objData{retries: 0} }) @@ -117,7 +117,7 @@ func Test_ReadyTracker_TryCancelTemplate_No_Retries(t *testing.T) { // Verify that TryCancelTemplate must be called enough times to remove all retries before canceling a template. func Test_ReadyTracker_TryCancelTemplate_Retries(t *testing.T) { - lister := fake.NewClientBuilder().WithRuntimeObjects(&convertedTemplate).Build() + lister := fake.NewClientBuilder().WithRuntimeObjects(convertedTemplate.DeepCopyObject()).Build() rt := newTracker(lister, false, false, false, func() objData { return objData{retries: 2} }) From 352ae726f09ae88c1b9b98fc3597362adb6e98ce Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 16 Nov 2023 22:28:20 +0000 Subject: [PATCH 40/42] rf: add set directly Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 42678933cb9..abd2d1844de 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -180,7 +180,7 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { c.gvksToDeleteFromCache.AddSet(finallyRemoved) c.danglingWatches.RemoveSet(finallyRemoved) - c.danglingWatches.AddSet(removeGVKFailures.Difference(c.danglingWatches)) + c.danglingWatches.AddSet(removeGVKFailures) } else { // defensively assume all watches that needed removal failed to be removed in the general error case // also assume whatever watches were dangling are still dangling. From a280fb7ec8a27cbea48af36e5a4f31198267aeec Mon Sep 17 00:00:00 2001 From: alex <8968914+acpana@users.noreply.github.com> Date: Wed, 22 Nov 2023 11:59:59 -0800 Subject: [PATCH 41/42] Apply suggestions from code review Co-authored-by: Rita Zhang Signed-off-by: alex <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/aggregator/aggregator.go | 2 +- pkg/readiness/ready_tracker.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index 0b8785e02f4..806450a3fca 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -63,7 +63,7 @@ func (b *GVKAgreggator) Remove(k Key) { } // Upsert stores an association between Key k and the list of GVKs -// and also the reverse associatoin between each GVK passed in and Key k. +// and also the reverse association between each GVK passed in and Key k. // Any old associations are dropped, unless they are included in the new list of // GVKs. func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) { diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index 2528559abb4..ae5130de1ae 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -734,7 +734,7 @@ func (t *Tracker) trackConstraintTemplates(ctx context.Context) error { // trackSyncSources sets expectations for cached data as specified by the singleton Config resource. // and any SyncSet resources present on the cluster. -// Works best effort and fails-open if the a resource cannot be fetched or does not exist. +// Works best effort and fails-open if a resource cannot be fetched or does not exist. func (t *Tracker) trackSyncSources(ctx context.Context) error { defer func() { t.config.ExpectationsDone() From c87e93428c4401c5b203dde9a94d43556e014194 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 22 Nov 2023 20:09:17 +0000 Subject: [PATCH 42/42] review suggestions Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 5 +++-- pkg/readiness/pruner/pruner.go | 2 +- pkg/readiness/pruner/pruner_test.go | 2 +- pkg/readiness/ready_tracker.go | 8 ++++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index abd2d1844de..84d1da5a389 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -405,8 +405,9 @@ func (c *CacheManager) manageCache(ctx context.Context) { // first make sure there is no drift between c.gvksToSync and watch manager if c.danglingWatches.Size() > 0 { - err := c.replaceWatchSet(ctx) - log.V(logging.DebugLevel).Info("error replacing watch set", "error", err) + if err := c.replaceWatchSet(ctx); err != nil { + log.V(logging.DebugLevel).Info("error replacing watch set", "error", err) + } } c.wipeCacheIfNeeded(ctx) diff --git a/pkg/readiness/pruner/pruner.go b/pkg/readiness/pruner/pruner.go index 8d864727ed8..f21ce5ef400 100644 --- a/pkg/readiness/pruner/pruner.go +++ b/pkg/readiness/pruner/pruner.go @@ -37,7 +37,7 @@ func (e *ExpectationsPruner) Start(ctx context.Context) error { // further manage the data sync expectations. return nil } - if e.tracker.SyncSourcesSatisfied() { + if e.tracker.SyncSetAndConfigSatisfied() { e.pruneUnwatchedGVKs() } } diff --git a/pkg/readiness/pruner/pruner_test.go b/pkg/readiness/pruner/pruner_test.go index cc0b8214ef1..32991d16b28 100644 --- a/pkg/readiness/pruner/pruner_test.go +++ b/pkg/readiness/pruner/pruner_test.go @@ -129,7 +129,7 @@ func Test_ExpectationsPruner_missedInformers(t *testing.T) { testutils.StartManager(ctx, t, testRes.manager) require.Eventually(t, func() bool { - return testRes.expectationsPruner.tracker.SyncSourcesSatisfied() + return testRes.expectationsPruner.tracker.SyncSetAndConfigSatisfied() }, timeout, tick, "waiting on sync sources to get satisfied") // As configMapGVK is absent from this syncset-a, the CacheManager will never observe configMapGVK diff --git a/pkg/readiness/ready_tracker.go b/pkg/readiness/ready_tracker.go index ae5130de1ae..94c86cadeeb 100644 --- a/pkg/readiness/ready_tracker.go +++ b/pkg/readiness/ready_tracker.go @@ -346,7 +346,7 @@ func (t *Tracker) Run(ctx context.Context) error { }) } grp.Go(func() error { - return t.trackSyncSources(gctx) + return t.trackConfigAndSyncSets(gctx) }) grp.Go(func() error { @@ -406,7 +406,7 @@ func (t *Tracker) Populated() bool { } // Returns whether both the Config and all SyncSet expectations have been Satisfied. -func (t *Tracker) SyncSourcesSatisfied() bool { +func (t *Tracker) SyncSetAndConfigSatisfied() bool { satisfied := t.config.Satisfied() if operations.HasValidationOperations() { satisfied = satisfied && t.syncsets.Satisfied() @@ -732,10 +732,10 @@ func (t *Tracker) trackConstraintTemplates(ctx context.Context) error { return nil } -// trackSyncSources sets expectations for cached data as specified by the singleton Config resource. +// trackConfigAndSyncSets sets expectations for cached data as specified by the singleton Config resource. // and any SyncSet resources present on the cluster. // Works best effort and fails-open if a resource cannot be fetched or does not exist. -func (t *Tracker) trackSyncSources(ctx context.Context) error { +func (t *Tracker) trackConfigAndSyncSets(ctx context.Context) error { defer func() { t.config.ExpectationsDone() log.V(logging.DebugLevel).Info("config expectations populated")