feat(kubernetes): add possibility to not add owner reference (#9794)

Introduce a new config flag to not add ownerReference to resources.

This is useful when using argoCD which will refuse deleting resources
if they have an ownerReference

Fix #9541

Signed-off-by: Charly Molter <charly.molter@konghq.com>

docs/generated/raw/kuma-cp.yaml

@@ -401,6 +401,10 @@ runtime:
       # RenewDeadline is the duration that the acting controlplane will retry
       # refreshing leadership before giving up. Default is 10 seconds.
       renewDeadline: 10s # ENV: KUMA_RUNTIME_KUBERNETES_LEADER_ELECTION_RENEW_DEADLINE
+    # SkipMeshOwnerReference is a flag that allows to skip adding Mesh owner reference to resources.
+    # If this is set to true, deleting a Mesh will not delete resources that belong to that Mesh.
+    # This can be useful when resources are managed in Argo CD where creation/deletion is managed there.
+    skipMeshOwnerReference: false # ENV: KUMA_RUNTIME_KUBERNETES_SKIP_MESH_OWNER_REFERENCE
   # Universal-specific configuration
   universal:
     # DataplaneCleanupAge defines how long Dataplane should be offline to be cleaned up by GC

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -401,6 +401,10 @@ runtime:
       # RenewDeadline is the duration that the acting controlplane will retry
       # refreshing leadership before giving up. Default is 10 seconds.
       renewDeadline: 10s # ENV: KUMA_RUNTIME_KUBERNETES_LEADER_ELECTION_RENEW_DEADLINE
+    # SkipMeshOwnerReference is a flag that allows to skip adding Mesh owner reference to resources.
+    # If this is set to true, deleting a Mesh will not delete resources that belong to that Mesh.
+    # This can be useful when resources are managed in Argo CD where creation/deletion is managed there.
+    skipMeshOwnerReference: false # ENV: KUMA_RUNTIME_KUBERNETES_SKIP_MESH_OWNER_REFERENCE
   # Universal-specific configuration
   universal:
     # DataplaneCleanupAge defines how long Dataplane should be offline to be cleaned up by GC

pkg/config/loader_test.go

@@ -231,6 +231,8 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.Runtime.Kubernetes.ClientConfig.BurstQps).To(Equal(100))
 			Expect(cfg.Runtime.Kubernetes.LeaderElection.LeaseDuration.Duration).To(Equal(199 * time.Second))
 			Expect(cfg.Runtime.Kubernetes.LeaderElection.RenewDeadline.Duration).To(Equal(99 * time.Second))
+			Expect(cfg.Runtime.Kubernetes.SkipMeshOwnerReference).To(BeTrue())
+
 			Expect(cfg.Runtime.Universal.DataplaneCleanupAge.Duration).To(Equal(1 * time.Hour))
 			Expect(cfg.Runtime.Universal.VIPRefreshInterval.Duration).To(Equal(10 * time.Second))
 
@@ -563,6 +565,7 @@ runtime:
     leaderElection:
       leaseDuration: 199s
       renewDeadline: 99s
+    skipMeshOwnerReference: true
 reports:
   enabled: false
 general:
@@ -886,6 +889,7 @@ tracing:
 				"KUMA_RUNTIME_KUBERNETES_CLIENT_CONFIG_BURST_QPS":                                          "100",
 				"KUMA_RUNTIME_KUBERNETES_LEADER_ELECTION_LEASE_DURATION":                                   "199s",
 				"KUMA_RUNTIME_KUBERNETES_LEADER_ELECTION_RENEW_DEADLINE":                                   "99s",
+				"KUMA_RUNTIME_KUBERNETES_SKIP_MESH_OWNER_REFERENCE":                                        "true",
 				"KUMA_RUNTIME_UNIVERSAL_DATAPLANE_CLEANUP_AGE":                                             "1h",
 				"KUMA_RUNTIME_UNIVERSAL_VIP_REFRESH_INTERVAL":                                              "10s",
 				"KUMA_GENERAL_TLS_CERT_FILE":                                                               "/tmp/cert",

pkg/config/plugins/runtime/k8s/config.go

@@ -149,6 +149,10 @@ type KubernetesRuntimeConfig struct {
 	ClientConfig ClientConfig `json:"clientConfig"`
 	// Kubernetes leader election configuration
 	LeaderElection LeaderElection `json:"leaderElection"`
+	// SkipMeshOwnerReference is a flag that allows to skip adding Mesh owner reference to resources.
+	// If this is set to true, deleting a Mesh will not delete resources that belong to that Mesh.
+	// This can be useful when resources are managed in Argo CD where creation/deletion is managed there.
+	SkipMeshOwnerReference bool `json:"skipMeshOwnerReference" envconfig:"kuma_runtime_kubernetes_skip_mesh_owner_reference"`
 }
 
 type ControllersConcurrency struct {

pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml

@@ -79,3 +79,4 @@ nodeTaintController:
   cniNamespace: kube-system
   enabled: false
 serviceAccountName: system:serviceaccount:kuma-system:kuma-control-plane
+skipMeshOwnerReference: false

pkg/plugins/runtime/k8s/plugin.go

@@ -348,11 +348,12 @@ func addMutators(mgr kube_ctrl.Manager, rt core_runtime.Runtime, converter k8s_c
 	}
 
 	ownerRefMutator := &k8s_webhooks.OwnerReferenceMutator{
-		Client:       mgr.GetClient(),
-		CoreRegistry: core_registry.Global(),
-		K8sRegistry:  k8s_registry.Global(),
-		Scheme:       mgr.GetScheme(),
-		Decoder:      kube_admission.NewDecoder(mgr.GetScheme()),
+		Client:                 mgr.GetClient(),
+		CoreRegistry:           core_registry.Global(),
+		K8sRegistry:            k8s_registry.Global(),
+		Scheme:                 mgr.GetScheme(),
+		Decoder:                kube_admission.NewDecoder(mgr.GetScheme()),
+		SkipMeshOwnerReference: rt.Config().Runtime.Kubernetes.SkipMeshOwnerReference,
 	}
 	mgr.GetWebhookServer().Register("/owner-reference-kuma-io-v1alpha1", &kube_webhook.Admission{Handler: ownerRefMutator})
 

pkg/plugins/runtime/k8s/webhooks/owner_reference_mutator.go

@@ -20,11 +20,12 @@ import (
 )
 
 type OwnerReferenceMutator struct {
-	Client       kube_client.Client
-	CoreRegistry core_registry.TypeRegistry
-	K8sRegistry  k8s_registry.TypeRegistry
-	Decoder      *admission.Decoder
-	Scheme       *kube_runtime.Scheme
+	Client                 kube_client.Client
+	CoreRegistry           core_registry.TypeRegistry
+	K8sRegistry            k8s_registry.TypeRegistry
+	Decoder                *admission.Decoder
+	Scheme                 *kube_runtime.Scheme
+	SkipMeshOwnerReference bool
 }
 
 func (m *OwnerReferenceMutator) Handle(ctx context.Context, req admission.Request) admission.Response {
@@ -54,6 +55,9 @@ func (m *OwnerReferenceMutator) Handle(ctx context.Context, req admission.Reques
 			return admission.Errored(http.StatusBadRequest, err)
 		}
 	default:
+		if m.SkipMeshOwnerReference {
+			return admission.Allowed("ignored. Configuration setup to ignore Mesh owner reference.")
+		}
 		// we need to also validate Mesh here because OwnerReferenceMutator is executed before validatingHandler
 		if err := core_mesh.ValidateMesh(obj.GetMesh(), coreRes.Descriptor().Scope); err.HasViolations() {
 			return convertValidationErrorOf(err, obj, obj.GetObjectMeta())

pkg/plugins/runtime/k8s/webhooks/owner_reference_mutator_test.go

@@ -10,56 +10,50 @@ import (
 	admissionv1 "k8s.io/api/admission/v1"
 	kube_meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kube_runtime "k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	kube_admission "sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	core_registry "github.com/kumahq/kuma/pkg/core/resources/registry"
-	mesh_k8s "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1"
-	"github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/pkg/model"
 	k8s_registry "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/pkg/registry"
 	"github.com/kumahq/kuma/pkg/plugins/runtime/k8s/webhooks"
 )
 
 var _ = Describe("OwnerReferenceMutator", func() {
-	createWebhook := func() webhook.AdmissionHandler {
-		return &webhooks.OwnerReferenceMutator{
-			Client:       k8sClient,
-			CoreRegistry: core_registry.Global(),
-			K8sRegistry:  k8s_registry.Global(),
-			Decoder:      decoder,
-			Scheme:       scheme,
-		}
-	}
-
-	createRequest := func(obj model.KubernetesObject, raw []byte) kube_admission.Request {
-		return kube_admission.Request{
-			AdmissionRequest: admissionv1.AdmissionRequest{
-				UID: "12345",
-				Object: kube_runtime.RawExtension{
-					Raw: raw,
-				},
-				Kind: kube_meta.GroupVersionKind{
-					Group:   obj.GetObjectKind().GroupVersionKind().Group,
-					Version: obj.GetObjectKind().GroupVersionKind().Version,
-					Kind:    obj.GetObjectKind().GroupVersionKind().Kind,
-				},
-			},
-		}
-	}
-
 	type testCase struct {
-		inputObject     string
-		expectedPatch   string
-		expectedMessage string
+		inputObject            string
+		expectedPatch          string
+		expectedMessage        string
+		skipMeshOwnerReference bool
+		ownerId                kube_meta.Object
 	}
 	DescribeTable("should add owner reference to resource owned by Mesh",
 		func(given testCase) {
-			tr := &mesh_k8s.TrafficRoute{}
-			err := json.Unmarshal([]byte(given.inputObject), tr)
-			Expect(err).ToNot(HaveOccurred())
-
-			wh := createWebhook()
-			r := wh.Handle(context.Background(), createRequest(tr, []byte(given.inputObject)))
+			if given.ownerId == nil {
+				given.ownerId = defaultMesh
+			}
+			k8sGroupVersionKind := kube_meta.GroupVersionKind{}
+			Expect(json.Unmarshal([]byte(given.inputObject), &k8sGroupVersionKind)).To(Succeed())
+			wh := &webhooks.OwnerReferenceMutator{
+				Client:                 k8sClient,
+				CoreRegistry:           core_registry.Global(),
+				K8sRegistry:            k8s_registry.Global(),
+				Decoder:                decoder,
+				Scheme:                 scheme,
+				SkipMeshOwnerReference: given.skipMeshOwnerReference,
+			}
+			req := kube_admission.Request{
+				AdmissionRequest: admissionv1.AdmissionRequest{
+					UID: "12345",
+					Object: kube_runtime.RawExtension{
+						Raw: []byte(given.inputObject),
+					},
+					Kind: kube_meta.GroupVersionKind{
+						Group:   k8sGroupVersionKind.Group,
+						Version: k8sGroupVersionKind.Version,
+						Kind:    k8sGroupVersionKind.Kind,
+					},
+				},
+			}
+			r := wh.Handle(context.Background(), req)
 			if given.expectedMessage != "" {
 				Expect(r.Result.Message).To(Equal(given.expectedMessage))
 			} else {
@@ -68,7 +62,7 @@ var _ = Describe("OwnerReferenceMutator", func() {
 			if given.expectedPatch != "" {
 				patch, err := json.Marshal(r.Patches)
 				Expect(err).ToNot(HaveOccurred())
-				Expect(string(patch)).To(MatchJSON(fmt.Sprintf(given.expectedPatch, defaultMesh.GetUID())))
+				Expect(string(patch)).To(MatchJSON(fmt.Sprintf(given.expectedPatch, given.ownerId.GetUID())))
 			} else {
 				Expect(r.Patches).To(BeNil())
 			}
@@ -141,10 +135,8 @@ var _ = Describe("OwnerReferenceMutator", func() {
             }`,
 			expectedMessage: "ignored. MeshService has a reference for Service",
 		}),
-	)
-
-	It("should add owner reference to resource owned by Dataplane", func() {
-		inputObject := `
+		Entry("should add owner reference to resource owned by Dataplane", testCase{
+			inputObject: `
             {
               "apiVersion": "kuma.io/v1alpha1",
               "kind": "DataplaneInsight",
@@ -154,25 +146,37 @@ var _ = Describe("OwnerReferenceMutator", func() {
                 "name": "dp-1",
                 "creationTimestamp": null
               }
-            }`
-		dpInsight := &mesh_k8s.DataplaneInsight{}
-		err := json.Unmarshal([]byte(inputObject), dpInsight)
-		Expect(err).ToNot(HaveOccurred())
-
-		dp := &mesh_k8s.Dataplane{
-			ObjectMeta: kube_meta.ObjectMeta{
-				Name:      "dp-1",
-				Namespace: "default",
-			},
-			Mesh: "default",
-		}
-		err = k8sClient.Create(context.Background(), dp)
-		Expect(err).ToNot(HaveOccurred())
-
-		wh := createWebhook()
-		r := wh.Handle(context.Background(), createRequest(dpInsight, []byte(inputObject)))
-
-		expectedPatch := fmt.Sprintf(`
+            }`,
+			expectedPatch: `
+            [
+              {
+                "op": "add",
+                "path": "/metadata/ownerReferences",
+                "value": [
+                  {
+                    "apiVersion": "kuma.io/v1alpha1",
+                    "kind": "Dataplane",
+                    "name": "dp-1",
+                    "uid": "%s"
+                  }
+                ]
+              }
+            ]`,
+			ownerId: dp1,
+		}),
+		Entry("should add owner reference to resource owned by Dataplane even with SkipMeshOwnerReference", testCase{
+			inputObject: `
+            {
+              "apiVersion": "kuma.io/v1alpha1",
+              "kind": "DataplaneInsight",
+              "mesh": "default",
+              "metadata": {
+                "namespace": "default",
+                "name": "dp-1",
+                "creationTimestamp": null
+              }
+            }`,
+			expectedPatch: `
             [
               {
                 "op": "add",
@@ -186,9 +190,23 @@ var _ = Describe("OwnerReferenceMutator", func() {
                   }
                 ]
               }
-            ]`, dp.GetUID())
-		patch, err := json.Marshal(r.Patches)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(string(patch)).To(MatchJSON(expectedPatch))
-	})
+            ]`,
+			ownerId:                dp1,
+			skipMeshOwnerReference: true,
+		}),
+		Entry("should ignore mesh owner reference", testCase{
+			inputObject: `
+            {
+              "apiVersion": "kuma.io/v1alpha1",
+              "kind": "TrafficRoute",
+              "metadata": {
+                "namespace": "example",
+                "name": "empty",
+                "creationTimestamp": null
+              }
+            }`,
+			skipMeshOwnerReference: true,
+			expectedMessage:        "ignored. Configuration setup to ignore Mesh owner reference.",
+		}),
+	)
 })

pkg/plugins/runtime/k8s/webhooks/webhook_suite_test.go

@@ -27,7 +27,18 @@ var (
 	testEnv     *envtest.Environment
 	k8sClient   client.Client
 	scheme      *kube_runtime.Scheme
-	defaultMesh *mesh_k8s.Mesh
+	defaultMesh = &mesh_k8s.Mesh{
+		ObjectMeta: kube_meta.ObjectMeta{
+			Name: "default",
+		},
+	}
+	dp1 = &mesh_k8s.Dataplane{
+		ObjectMeta: kube_meta.ObjectMeta{
+			Name:      "dp-1",
+			Namespace: "default",
+		},
+		Mesh: "default",
+	}
 )
 
 var _ = BeforeSuite(func() {
@@ -52,13 +63,9 @@ var _ = BeforeSuite(func() {
 	Expect(k8sClient).ToNot(BeNil())
 
 	// create default mesh
-	defaultMesh = &mesh_k8s.Mesh{
-		ObjectMeta: kube_meta.ObjectMeta{
-			Name: "default",
-		},
-	}
 	err = k8sClient.Create(context.Background(), defaultMesh)
 	Expect(err).ToNot(HaveOccurred())
+	Expect(k8sClient.Create(context.Background(), dp1)).To(Succeed())
 })
 
 var _ = AfterSuite(func() {