[release-1.6] Opt-out from cluster-wide default node selector (#1799) (…

…#1816) The hyperconverged cluster operator defines its own node selectors APIs that schedules deployments and daemon sets on nodes according to infra and workload classification. Opting out from cluster-wide default node selector on Openshift setting a specific annotation (openshift.io/node-selector=) to prevent conflicts. Bug-Url: https://bugzilla.redhat.com/show_bug.cgi?id=2055950 This is a manual cherry-pick of #1799 Signed-off-by: Simone Tiraboschi <stirabos@redhat.com>
kubevirt · Mar 14, 2022 · 4700fae · 4700fae
1 parent c986a61
commit 4700fae
Show file tree

Hide file tree

Showing 15 changed files with 294 additions and 34 deletions.
diff --git a/cmd/hyperconverged-cluster-operator/main.go b/cmd/hyperconverged-cluster-operator/main.go
@@ -180,6 +180,7 @@ func main() {
 func getNewManagerCache(operatorNamespace string) cache.NewCacheFunc {
 	namespaceSelector := fields.Set{"metadata.namespace": operatorNamespace}.AsSelector()
 	labelSelector := labels.Set{hcoutil.AppLabel: hcoutil.HyperConvergedName}.AsSelector()
+	labelSelectorForNamespace := labels.Set{hcoutil.KubernetesMetadataName: operatorNamespace}.AsSelector()
 	return cache.BuilderWithOptions(
 		cache.Options{
 			SelectorsByObject: cache.SelectorsByObject{
@@ -219,6 +220,9 @@ func getNewManagerCache(operatorNamespace string) cache.NewCacheFunc {
 				&imagev1.ImageStream{}: {
 					Label: labelSelector,
 				},
+				&corev1.Namespace{}: {
+					Label: labelSelectorForNamespace,
+				},
 			},
 		},
 	)

diff --git a/deploy/cluster_role.yaml b/deploy/cluster_role.yaml
@@ -466,6 +466,16 @@ rules:
   - get
   - list
   - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update
 - apiGroups:
   - apps
   resources:

diff --git a/...verged/1.6.0/manifests/kubevirt-hyperconverged-operator.v1.6.0.clusterserviceversion.yaml b/...verged/1.6.0/manifests/kubevirt-hyperconverged-operator.v1.6.0.clusterserviceversion.yaml
@@ -298,6 +298,16 @@ spec:
           - get
           - list
           - delete
+        - apiGroups:
+          - ""
+          resources:
+          - namespaces
+          verbs:
+          - get
+          - list
+          - watch
+          - patch
+          - update
         - apiGroups:
           - apps
           resources:
@@ -3270,7 +3280,7 @@ spec:
     generateName: mutate-ns-hco.kubevirt.io
     objectSelector:
       matchLabels:
-        name: kubevirt-hyperconverged
+        kubernetes.io/metadata.name: kubevirt-hyperconverged
     rules:
     - apiGroups:
       - ""

diff --git a/...verged/1.6.0/manifests/kubevirt-hyperconverged-operator.v1.6.0.clusterserviceversion.yaml b/...verged/1.6.0/manifests/kubevirt-hyperconverged-operator.v1.6.0.clusterserviceversion.yaml
@@ -9,7 +9,7 @@ metadata:
     categories: OpenShift Optional
     certified: "false"
     containerImage: quay.io/kubevirt/hyperconverged-cluster-operator:1.6.0-unstable
-    createdAt: "2022-02-16 10:12:22"
+    createdAt: "2022-03-13 21:00:31"
     description: A unified operator deploying and controlling KubeVirt and its supporting
       operators with opinionated defaults
     operatorframework.io/initialization-resource: '{"apiVersion":"hco.kubevirt.io/v1beta1","kind":"HyperConverged","metadata":{"annotations":{"deployOVS":"false"},"name":"kubevirt-hyperconverged","namespace":"kubevirt-hyperconverged"},"spec":{}}'
@@ -298,6 +298,16 @@ spec:
           - get
           - list
           - delete
+        - apiGroups:
+          - ""
+          resources:
+          - namespaces
+          verbs:
+          - get
+          - list
+          - watch
+          - patch
+          - update
         - apiGroups:
           - apps
           resources:
@@ -3270,7 +3280,7 @@ spec:
     generateName: mutate-ns-hco.kubevirt.io
     objectSelector:
       matchLabels:
-        name: kubevirt-hyperconverged
+        kubernetes.io/metadata.name: kubevirt-hyperconverged
     rules:
     - apiGroups:
       - ""

diff --git a/pkg/components/components.go b/pkg/components/components.go
@@ -452,6 +452,11 @@ func GetClusterPermissions() []rbacv1.PolicyRule {
 			Resources: stringListToSlice("endpoints"),
 			Verbs:     stringListToSlice("get", "list", "delete"),
 		},
+		{
+			APIGroups: emptyAPIGroup,
+			Resources: stringListToSlice("namespaces"),
+			Verbs:     stringListToSlice("get", "list", "watch", "patch", "update"),
+		},
 		{
 			APIGroups: stringListToSlice("apps"),
 			Resources: stringListToSlice("deployments", "replicasets"),
@@ -776,7 +781,7 @@ func GetCSVBase(params *CSVBaseParams) *csvv1alpha1.ClusterServiceVersion {
 		FailurePolicy:           &failurePolicy,
 		TimeoutSeconds:          &webhookTimeout,
 		ObjectSelector: &metav1.LabelSelector{
-			MatchLabels: map[string]string{"name": params.Namespace},
+			MatchLabels: map[string]string{util.KubernetesMetadataName: params.Namespace},
 		},
 		Rules: []admissionregistrationv1.RuleWithOperations{
 			{

diff --git a/pkg/controller/commonTestUtils/testUtils.go b/pkg/controller/commonTestUtils/testUtils.go
@@ -57,6 +57,14 @@ func NewHco() *hcov1beta1.HyperConverged {
 	return hco
 }
 
+func NewHcoNamespace() *corev1.Namespace {
+	return &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: Namespace,
+		},
+	}
+}
+
 func NewReq(inst *hcov1beta1.HyperConverged) *common.HcoRequest {
 	return &common.HcoRequest{
 		Request:      TestRequest,

diff --git a/pkg/controller/hyperconverged/hyperconverged_controller.go b/pkg/controller/hyperconverged/hyperconverged_controller.go
@@ -178,6 +178,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, ci hcoutil.ClusterInfo) er
 			&routev1.Route{},
 			&consolev1.ConsoleCLIDownload{},
 			&imagev1.ImageStream{},
+			&corev1.Namespace{},
 		}...)
 	}
 

diff --git a/pkg/controller/hyperconverged/hyperconverged_controller_test.go b/pkg/controller/hyperconverged/hyperconverged_controller_test.go
@@ -79,10 +79,15 @@ var _ = Describe("HyperconvergedController", func() {
 	Describe("Reconcile HyperConverged", func() {
 		Context("HCO Lifecycle", func() {
 
+			var (
+				hcoNamespace *corev1.Namespace
+			)
+
 			BeforeEach(func() {
 				_ = os.Setenv("VIRTIOWIN_CONTAINER", commonTestUtils.VirtioWinImage)
 				_ = os.Setenv("OPERATOR_NAMESPACE", namespace)
 				_ = os.Setenv(hcoutil.HcoKvIoVersionName, version.Version)
+				hcoNamespace = commonTestUtils.NewHcoNamespace()
 			})
 
 			It("should handle not found", func() {
@@ -106,7 +111,7 @@ var _ = Describe("HyperconvergedController", func() {
 						Conditions: []metav1.Condition{},
 					},
 				}
-				cl := commonTestUtils.InitClient([]runtime.Object{hco})
+				cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 				r := initReconciler(cl, nil)
 
 				// Do the reconcile
@@ -137,13 +142,12 @@ var _ = Describe("HyperconvergedController", func() {
 			})
 
 			It("should create all managed resources", func() {
-
 				hco := commonTestUtils.NewHco()
 				hco.Spec.FeatureGates = hcov1beta1.HyperConvergedFeatureGates{
 					WithHostPassthroughCPU: true,
 				}
 
-				cl := commonTestUtils.InitClient([]runtime.Object{hco})
+				cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 
 				r := initReconciler(cl, nil)
 
@@ -511,7 +515,7 @@ var _ = Describe("HyperconvergedController", func() {
 				existingResource.Spec.Infra.NodePlacement.NodeSelector["key1"] = "BADvalue1"
 				existingResource.Spec.Workloads.NodePlacement.NodeSelector["key2"] = "BADvalue2"
 
-				cl := commonTestUtils.InitClient([]runtime.Object{hco, existingResource})
+				cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco, existingResource})
 				r := initReconciler(cl, nil)
 
 				// mock a reconciliation triggered by a change in secondary CR
@@ -571,7 +575,7 @@ var _ = Describe("HyperconvergedController", func() {
 				existingResource.Spec.Infra.NodePlacement.NodeSelector["key1"] = "BADvalue1"
 				existingResource.Spec.Workloads.NodePlacement.NodeSelector["key2"] = "BADvalue2"
 
-				cl := commonTestUtils.InitClient([]runtime.Object{hco, existingResource})
+				cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco, existingResource})
 				r := initReconciler(cl, nil)
 
 				counterValueBefore, err := metrics.HcoMetrics.GetOverwrittenModificationsCount(existingResource.Kind, existingResource.Name)
@@ -756,7 +760,7 @@ var _ = Describe("HyperconvergedController", func() {
 
 			It("Should not be ready if one of the operands is returns error, on create", func() {
 				hco := commonTestUtils.NewHco()
-				cl := commonTestUtils.InitClient([]runtime.Object{hco})
+				cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 				cl.InitiateCreateErrors(func(obj client.Object) error {
 					if _, ok := obj.(*cdiv1beta1.CDI); ok {
 						return errors.New("fake create error")
@@ -2212,9 +2216,11 @@ var _ = Describe("HyperconvergedController", func() {
 
 		Context("Detection of a tainted configuration", func() {
 			var (
-				hco *hcov1beta1.HyperConverged
+				hcoNamespace *corev1.Namespace
+				hco          *hcov1beta1.HyperConverged
 			)
 			BeforeEach(func() {
+				hcoNamespace = commonTestUtils.NewHcoNamespace()
 				hco = commonTestUtils.NewHco()
 				UpdateVersion(&hco.Status, hcoVersionName, version.Version)
 				_ = os.Setenv(hcoutil.HcoKvIoVersionName, version.Version)
@@ -2236,7 +2242,7 @@ var _ = Describe("HyperconvergedController", func() {
 					err := metrics.HcoMetrics.SetUnsafeModificationCount(0, common.JSONPatchKVAnnotationName)
 					Expect(err).To(BeNil())
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					By("Reconcile", func() {
@@ -2290,7 +2296,7 @@ var _ = Describe("HyperconvergedController", func() {
 					err := metrics.HcoMetrics.SetUnsafeModificationCount(5, common.JSONPatchKVAnnotationName)
 					Expect(err).To(BeNil())
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					// Do the reconcile
@@ -2344,7 +2350,7 @@ var _ = Describe("HyperconvergedController", func() {
 						]`,
 					}
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					By("Reconcile", func() {
@@ -2396,7 +2402,7 @@ var _ = Describe("HyperconvergedController", func() {
 					err := metrics.HcoMetrics.SetUnsafeModificationCount(0, common.JSONPatchCDIAnnotationName)
 					Expect(err).To(BeNil())
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					By("Reconcile", func() {
@@ -2454,7 +2460,7 @@ var _ = Describe("HyperconvergedController", func() {
 					err := metrics.HcoMetrics.SetUnsafeModificationCount(5, common.JSONPatchCDIAnnotationName)
 					Expect(err).To(BeNil())
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					// Do the reconcile
@@ -2501,7 +2507,7 @@ var _ = Describe("HyperconvergedController", func() {
 						common.JSONPatchKVAnnotationName: `[{`,
 					}
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					By("Reconcile", func() {
@@ -2552,7 +2558,7 @@ var _ = Describe("HyperconvergedController", func() {
 					err := metrics.HcoMetrics.SetUnsafeModificationCount(0, common.JSONPatchCNAOAnnotationName)
 					Expect(err).To(BeNil())
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					By("Reconcile", func() {
@@ -2606,7 +2612,7 @@ var _ = Describe("HyperconvergedController", func() {
 					err := metrics.HcoMetrics.SetUnsafeModificationCount(5, common.JSONPatchCNAOAnnotationName)
 					Expect(err).To(BeNil())
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					// Do the reconcile
@@ -2645,7 +2651,7 @@ var _ = Describe("HyperconvergedController", func() {
 					err := metrics.HcoMetrics.SetUnsafeModificationCount(5, common.JSONPatchCNAOAnnotationName)
 					Expect(err).To(BeNil())
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					By("Reconcile", func() {
@@ -2718,7 +2724,7 @@ var _ = Describe("HyperconvergedController", func() {
 					err = metrics.HcoMetrics.SetUnsafeModificationCount(0, common.JSONPatchCNAOAnnotationName)
 					Expect(err).To(BeNil())
 
-					cl := commonTestUtils.InitClient([]runtime.Object{hco})
+					cl := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 					r := initReconciler(cl, nil)
 
 					By("Reconcile", func() {

diff --git a/pkg/controller/hyperconverged/testUtils_test.go b/pkg/controller/hyperconverged/testUtils_test.go
@@ -104,6 +104,7 @@ func validateOperatorCondition(r *ReconcileHyperConverged, status metav1.Conditi
 }
 
 type BasicExpected struct {
+	namespace            *corev1.Namespace
 	hco                  *hcov1beta1.HyperConverged
 	pc                   *schedulingv1.PriorityClass
 	kvStorageConfig      *corev1.ConfigMap
@@ -127,6 +128,7 @@ type BasicExpected struct {
 
 func (be BasicExpected) toArray() []runtime.Object {
 	return []runtime.Object{
+		be.namespace,
 		be.hco,
 		be.pc,
 		be.kvStorageConfig,
@@ -185,6 +187,15 @@ func getBasicDeployment() *BasicExpected {
 
 	components.GetOperatorCR().Spec.DeepCopyInto(&res.hco.Spec)
 
+	res.namespace = &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: hco.Namespace,
+			Annotations: map[string]string{
+				hcoutil.OpenshiftNodeSelectorAnn: "",
+			},
+		},
+	}
+
 	res.pc = operands.NewKubeVirtPriorityClass(hco)
 	res.mService = operands.NewMetricsService(hco, namespace)
 	res.serviceMonitor = operands.NewServiceMonitor(hco, namespace)

diff --git a/pkg/controller/operands/imageStream_test.go b/pkg/controller/operands/imageStream_test.go
@@ -119,10 +119,11 @@ var _ = Describe("imageStream tests", func() {
 				return testFilesLocation
 			}
 
+			hcoNamespace := commonTestUtils.NewHcoNamespace()
 			hco := commonTestUtils.NewHco()
 			hco.Spec.FeatureGates.EnableCommonBootImageImport = true
 			eventEmitter := commonTestUtils.NewEventEmitterMock()
-			cli := commonTestUtils.InitClient([]runtime.Object{hco})
+			cli := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 			handler := NewOperandHandler(cli, commonTestUtils.GetScheme(), true, eventEmitter)
 			handler.FirstUseInitiation(commonTestUtils.GetScheme(), true, hco)
 
@@ -183,8 +184,9 @@ var _ = Describe("imageStream tests", func() {
 				return testFilesLocation
 			}
 
+			hcoNamespace := commonTestUtils.NewHcoNamespace()
 			hco := commonTestUtils.NewHco()
-			cli := commonTestUtils.InitClient([]runtime.Object{hco})
+			cli := commonTestUtils.InitClient([]runtime.Object{hcoNamespace, hco})
 
 			eventEmitter := commonTestUtils.NewEventEmitterMock()
 			handler := NewOperandHandler(cli, commonTestUtils.GetScheme(), true, eventEmitter)