[BUGFIX] - Namespace Cache (#1350)

[BUGFIX] - Defer to Lookup
For some as yet unknown, but probably reason on my part the cache appears to loose items. Before we would warn on this issue, at the very least we can defer to a direct lookup, getting the current state from the api itself.

pkg/controller/configuration/controller.go

@@ -173,8 +173,7 @@ func (c *Controller) Add(mgr manager.Manager) error {
 	}
 
 	c.cc = mgr.GetClient()
-
-	c.cache = pcache.New(12*time.Hour, 10*time.Minute)
+	c.cache = pcache.New(24*time.Hour, 10*time.Minute)
 	c.cache.OnEvicted(func(key string, _ interface{}) {
 		// ensure we have some logging for when a namespace is evicted
 		log.WithField("namespace", key).Warn("evicted namespace from cache")

pkg/controller/configuration/ensure.go

@@ -444,13 +444,12 @@ func (c *Controller) ensureProviderReady(configuration *terraformv1alpha1.Config
 
 		// @step: ensure we are permitted to use the provider
 		if provider.Spec.Selector != nil {
-			value, found := c.cache.Get(configuration.Namespace)
-			if !found {
-				cond.Failed(errors.New("namespace not found"), "Failed to retrieve the namespace from the cache")
+			namespace, err := c.getNamespaceFromCache(ctx, configuration.Namespace)
+			if err != nil {
+				cond.Failed(err, "Failed to retrieve the namespace from the cache")
 
 				return reconcile.Result{RequeueAfter: 30 * time.Second}, nil
 			}
-			namespace := value.(*v1.Namespace)
 
 			// @step: ensure we have match the selector of the provider - i.e our namespace and resource labels must match
 			match, err := kubernetes.IsSelectorMatch(*provider.Spec.Selector, configuration.GetLabels(), namespace.GetLabels())
@@ -483,12 +482,11 @@ func (c *Controller) ensurePolicyDefaultsExist(configuration *terraformv1alpha1.
 			return reconcile.Result{}, nil
 		}
 
-		// @step: we need to retrieve the namespace from the cache
-		namespace, found := c.cache.Get(configuration.Namespace)
-		if !found {
-			log.WithFields(log.Fields{
-				"namespace": configuration.Namespace,
-			}).Warn("namespace not found in the cache, this will cause issues on selector policies")
+		namespace, err := c.getNamespaceFromCache(ctx, configuration.Namespace)
+		if err != nil {
+			cond.Failed(err, "Failed to retrieve the namespace")
+
+			return reconcile.Result{RequeueAfter: 30 * time.Second}, nil
 		}
 
 		var list []string
@@ -530,7 +528,7 @@ func (c *Controller) ensurePolicyDefaultsExist(configuration *terraformv1alpha1.
 						return reconcile.Result{RequeueAfter: 30 * time.Second}, nil
 					}
 
-					match, err := x.Selector.IsLabelsMatch(namespace.(*v1.Namespace))
+					match, err := x.Selector.IsLabelsMatch(namespace)
 					if err != nil {
 						cond.Failed(err, "Failed to check against the policy: %q", state.policies.Items[i].Name)
 

pkg/controller/configuration/helper.go

@@ -22,6 +22,9 @@ import (
 	"fmt"
 	"strings"
 
+	log "github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+
 	terraformv1alpha1 "github.com/appvia/terranetes-controller/pkg/apis/terraform/v1alpha1"
 	"github.com/appvia/terranetes-controller/pkg/utils/jobs"
 	"github.com/appvia/terranetes-controller/pkg/utils/kubernetes"
@@ -38,6 +41,32 @@ func GetTerraformImage(configuration *terraformv1alpha1.Configuration, image str
 	return fmt.Sprintf("%s:%s", e[0], configuration.Spec.TerraformVersion)
 }
 
+// getNamespaceFromCache is responsible for retrieving the namespace from the cache, or deferring
+// to a direct lookup if it's not found
+func (c *Controller) getNamespaceFromCache(ctx context.Context, name string) (*v1.Namespace, error) {
+	item, found := c.cache.Get(name)
+	if found {
+		return item.(*v1.Namespace), nil
+	}
+
+	log.WithFields(log.Fields{
+		"namespace": name,
+	}).Warn("namespace not found in the cache, deferring to a direct lookup")
+
+	namespace := &v1.Namespace{}
+	namespace.Name = name
+
+	if found, err := kubernetes.GetIfExists(ctx, c.cc, namespace); err != nil {
+		return nil, err
+	} else if !found {
+		return nil, fmt.Errorf("namespace %s not found", name)
+	}
+
+	c.cache.SetDefault(name, namespace)
+
+	return namespace, nil
+}
+
 // CreateWatcher is responsible for ensuring the logger is running in the application namespace
 func (c Controller) CreateWatcher(ctx context.Context, configuration *terraformv1alpha1.Configuration, stage string) error {
 	watcher := jobs.New(configuration, nil).NewJobWatch(c.ControllerNamespace, stage, c.ExecutorImage)