Merge pull request #35 from appuio/allow-setting-default-nodesel

Add mutating webhook to add APPUiO node selectors to pods instead of using `openshift.io/node-selector`

README.md

@@ -64,23 +64,35 @@ EOF
 kubectl apply -f ./config/webhook/manifests.yaml
 
 # On BSD/MacOS base64 must be called without `-w0`
+cert="$(base64 -w0 ./webhook-certs/tls.crt)"
 
-kubectl patch validatingwebhookconfiguration validating-webhook-configuration \
-  --type=json -p '[
+patch_tmpl='.webhooks | keys[] as $i | [
     {
       "op": "add",
-      "path": "/webhooks/0/clientConfig/caBundle",
-      "value": "'"$(base64 -w0 ./webhook-certs/tls.crt)"'"
+      "path": "/webhooks/\($i)/clientConfig/caBundle",
+      "value": "'"$cert"'"
     },
     {
       "op": "replace",
-      "path": "/webhooks/0/clientConfig/service/namespace",
+      "path": "/webhooks/\($i)/clientConfig/service/namespace",
       "value": "default"
     },
     {
       "op": "replace",
-      "path": "/webhooks/0/clientConfig/service/port",
+      "path": "/webhooks/\($i)/clientConfig/service/port",
       "value": 9443
     }
   ]'
+
+patches=$(kubectl get validatingwebhookconfiguration validating-webhook-configuration -ojson \
+  | jq -rc "$patch_tmpl")
+while read -r patch; do
+   kubectl patch validatingwebhookconfiguration validating-webhook-configuration --type=json -p "${patch}"
+done <<< "$patches"
+
+patches=$(kubectl get mutatingwebhookconfigurations mutating-webhook-configuration -ojson \
+  | jq -rc "$patch_tmpl")
+while read -r patch; do
+   kubectl patch mutatingwebhookconfigurations mutating-webhook-configuration --type=json -p "${patch}"
+done <<< "$patches"
 ```

config.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"errors"
 	"os"
 
+	"go.uber.org/multierr"
 	"sigs.k8s.io/yaml"
 )
 
@@ -23,13 +25,10 @@ type Config struct {
 	PrivilegedUsers        []string
 	PrivilegedClusterRoles []string
 
-	// AllowedNodeSelectors is a map of allowed node selectors.
-	// Both the key and the value are anchored regexes.
-	AllowedNodeSelectors map[string]string
-
-	// NamespaceDenyEmptyNodeSelector is a flag to enable or disable the rejection of empty node selectors on namespaces.
-	// If true this will reject a { "openshift.io/node-selector": "" } annotation.
-	NamespaceDenyEmptyNodeSelector bool
+	// DefaultNodeSelector are the default node selectors to add to pods if not set from namespace annotation
+	DefaultNodeSelector map[string]string
+	// DefaultNamespaceNodeSelectorAnnotation is the annotation used to set the default node selector for pods in this namespace
+	DefaultNamespaceNodeSelectorAnnotation string
 }
 
 func ConfigFromFile(path string) (Config, error) {
@@ -40,3 +39,16 @@ func ConfigFromFile(path string) (Config, error) {
 	var c Config
 	return c, yaml.Unmarshal(raw, &c, yaml.DisallowUnknownFields)
 }
+
+func (c Config) Validate() error {
+	var errs []error
+
+	if c.OrganizationLabel == "" {
+		errs = append(errs, errors.New("OrganizationLabel must not be empty"))
+	}
+	if c.MemoryPerCoreLimit == "" {
+		errs = append(errs, errors.New("MemoryPerCoreLimit must not be empty"))
+	}
+
+	return multierr.Combine(errs...)
+}

config.yaml

@@ -13,9 +13,9 @@ PrivilegedGroups: []
 PrivilegedUsers: []
 PrivilegedClusterRoles:
   - cluster-admin
-# Key and values are anchored regexes
-AllowedNodeSelectors:
-  appuio.io/node-class: flex|plus
-# NamespaceDenyEmptyNodeSelector is a flag to enable or disable the rejection of empty node selectors on namespaces.
-# If true this will reject a { "openshift.io/node-selector": "" } annotation.
-NamespaceDenyEmptyNodeSelector: false
+
+# Default node selectors to add to pods in this namespace
+DefaultNamespaceNodeSelectorAnnotation: appuio.io/default-node-selector
+# Default node selectors to add to pods if not set from namespace annotation
+DefaultNodeSelector:
+  appuio.io/node-class: plus

config/webhook/manifests.yaml

@@ -1,20 +1,20 @@
 ---
 apiVersion: admissionregistration.k8s.io/v1
-kind: ValidatingWebhookConfiguration
+kind: MutatingWebhookConfiguration
 metadata:
   creationTimestamp: null
-  name: validating-webhook-configuration
+  name: mutating-webhook-configuration
 webhooks:
   - admissionReviewVersions:
       - v1
     clientConfig:
       service:
         name: webhook-service
         namespace: system
-        path: /validate-namespace-node-selector
+        path: /mutate-pod-node-selector
     failurePolicy: Fail
     matchPolicy: Equivalent
-    name: validate-namespace-node-selector.appuio.io
+    name: mutate-pod-node-selector.appuio.io
     rules:
       - apiGroups:
           - ""
@@ -24,8 +24,15 @@ webhooks:
           - CREATE
           - UPDATE
         resources:
-          - namespaces
+          - pods
     sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: validating-webhook-configuration
+webhooks:
   - admissionReviewVersions:
       - v1
     clientConfig:
@@ -48,29 +55,3 @@ webhooks:
           - '*'
         scope: Namespaced
     sideEffects: None
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: webhook-service
-        namespace: system
-        path: /validate-workload-node-selector
-    failurePolicy: Fail
-    matchPolicy: Equivalent
-    name: validate-workload-node-selector.appuio.io
-    rules:
-      - apiGroups:
-          - '*'
-        apiVersions:
-          - v1
-        operations:
-          - CREATE
-          - UPDATE
-        resources:
-          - pods
-          - daemonsets
-          - deployments
-          - jobs
-          - statefulsets
-          - cronjobs
-    sideEffects: None

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/minio/pkg v1.4.5
 	github.com/stretchr/testify v1.8.0
 	go.uber.org/multierr v1.8.0
+	gomodules.xyz/jsonpatch/v2 v2.2.0
 	gopkg.in/inf.v0 v0.9.1
 	k8s.io/api v0.25.2
 	k8s.io/apimachinery v0.25.2
@@ -72,7 +73,6 @@ require (
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
 	golang.org/x/tools v0.1.12 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

main.go

@@ -17,7 +17,6 @@ import (
 	"github.com/appuio/appuio-cloud-agent/controllers"
 	"github.com/appuio/appuio-cloud-agent/ratio"
 	"github.com/appuio/appuio-cloud-agent/skipper"
-	"github.com/appuio/appuio-cloud-agent/validate"
 	"github.com/appuio/appuio-cloud-agent/webhooks"
 )
 
@@ -63,6 +62,10 @@ func main() {
 		setupLog.Error(err, "unable to read config file")
 		os.Exit(1)
 	}
+	if err := conf.Validate(); err != nil {
+		setupLog.Error(err, "invalid configuration")
+		os.Exit(1)
+	}
 
 	ctx := ctrl.SetupSignalHandler()
 
@@ -82,14 +85,17 @@ func main() {
 
 	registerRatioController(mgr, conf.MemoryPerCoreLimit, conf.OrganizationLabel)
 
+	// Currently unused, but will be used for the next kyverno replacements
 	psk := &skipper.PrivilegedUserSkipper{
 		Client: mgr.GetClient(),
 
 		PrivilegedUsers:        conf.PrivilegedUsers,
 		PrivilegedGroups:       conf.PrivilegedGroups,
 		PrivilegedClusterRoles: conf.PrivilegedClusterRoles,
 	}
-	registerNodeSelectorValidationWebhooks(mgr, psk, conf)
+	_ = psk
+
+	registerNodeSelectorValidationWebhooks(mgr, conf)
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to setup health endpoint")
@@ -107,26 +113,12 @@ func main() {
 	}
 }
 
-func registerNodeSelectorValidationWebhooks(mgr ctrl.Manager, skipper skipper.Skipper, conf Config) {
-	ans := &validate.AllowedLabels{}
-	for k, v := range conf.AllowedNodeSelectors {
-		if err := ans.Add(k, v); err != nil {
-			setupLog.Error(err, "unable to add allowed node selector")
-			os.Exit(1)
-		}
-	}
-
-	mgr.GetWebhookServer().Register("/validate-namespace-node-selector", &webhook.Admission{
-		Handler: &webhooks.NamespaceNodeSelectorValidator{
-			Skipper:               skipper,
-			AllowedNodeSelectors:  ans,
-			DenyEmptyNodeSelector: conf.NamespaceDenyEmptyNodeSelector,
-		},
-	})
-	mgr.GetWebhookServer().Register("/validate-workload-node-selector", &webhook.Admission{
-		Handler: &webhooks.WorkloadNodeSelectorValidator{
-			Skipper:              skipper,
-			AllowedNodeSelectors: ans,
+func registerNodeSelectorValidationWebhooks(mgr ctrl.Manager, conf Config) {
+	mgr.GetWebhookServer().Register("/mutate-pod-node-selector", &webhook.Admission{
+		Handler: &webhooks.PodNodeSelectorMutator{
+			Skipper:             skipper.StaticSkipper{ShouldSkip: false},
+			Client:              mgr.GetClient(),
+			DefaultNodeSelector: conf.DefaultNodeSelector,
 		},
 	})
 }