test(e2e): Add test for NFD addon (#389)

pkg/handlers/generic/lifecycle/nfd/handler.go

@@ -103,7 +103,13 @@ func (n *DefaultNFD) AfterControlPlaneInitialized(
 		log.Error(err, "failed to apply NFD ConfigMap for cluster")
 		return
 	}
-	err = utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, n.client, &req.Cluster, cm)
+	err = utils.EnsureCRSForClusterFromConfigMaps(
+		ctx,
+		cm.Name+"-"+req.Cluster.Name,
+		n.client,
+		&req.Cluster,
+		cm,
+	)
 	if err != nil {
 		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
 		log.Error(err, "failed to apply NFD ClusterResourceSet for cluster")

test/e2e/addon_helpers.go

@@ -0,0 +1,56 @@
+//go:build e2e
+
+// Copyright 2024 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/framework"
+
+	"github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1"
+)
+
+type WaitForAddonsToBeReadyInWorkloadClusterInput struct {
+	AddonsConfig                v1alpha1.Addons
+	WorkloadCluster             *capiv1.Cluster
+	ClusterProxy                framework.ClusterProxy
+	DeploymentIntervals         []interface{}
+	DaemonSetIntervals          []interface{}
+	HelmReleaseIntervals        []interface{}
+	ClusterResourceSetIntervals []interface{}
+}
+
+func WaitForAddonsToBeReadyInWorkloadCluster(
+	ctx context.Context,
+	input WaitForAddonsToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
+) {
+	WaitForCNIToBeReadyInWorkloadCluster(
+		ctx,
+		WaitForCNIToBeReadyInWorkloadClusterInput{
+			CNI:                         input.AddonsConfig.CNI,
+			WorkloadCluster:             input.WorkloadCluster,
+			ClusterProxy:                input.ClusterProxy,
+			DeploymentIntervals:         input.DeploymentIntervals,
+			DaemonSetIntervals:          input.DaemonSetIntervals,
+			HelmReleaseIntervals:        input.HelmReleaseIntervals,
+			ClusterResourceSetIntervals: input.ClusterResourceSetIntervals,
+		},
+	)
+
+	WaitForNFDToBeReadyInWorkloadCluster(
+		ctx,
+		WaitForNFDToBeReadyInWorkloadClusterInput{
+			NFD:                         input.AddonsConfig.NFD,
+			WorkloadCluster:             input.WorkloadCluster,
+			ClusterProxy:                input.ClusterProxy,
+			DeploymentIntervals:         input.DeploymentIntervals,
+			DaemonSetIntervals:          input.DaemonSetIntervals,
+			HelmReleaseIntervals:        input.HelmReleaseIntervals,
+			ClusterResourceSetIntervals: input.ClusterResourceSetIntervals,
+		},
+	)
+}

test/e2e/clusterresourceset_helpers.go

@@ -7,12 +7,16 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type waitForClusterResourceSetToApplyResourcesInClusterInput struct {
@@ -31,7 +35,7 @@ func waitForClusterResourceSetToApplyResourcesInCluster(
 		ctx, types.NamespacedName{Name: input.name, Namespace: input.cluster.Namespace}, crs,
 	)).To(Succeed())
 
-	framework.WaitForClusterResourceSetToApplyResources(
+	waitForClusterResourceSetToApplyResources(
 		ctx,
 		framework.WaitForClusterResourceSetToApplyResourcesInput{
 			ClusterProxy:       input.clusterProxy,
@@ -41,3 +45,83 @@ func waitForClusterResourceSetToApplyResourcesInCluster(
 		input.intervals...,
 	)
 }
+
+// Copied from upstream to fix bug checking bindings. Need to contribute fix back upstream and use it from there once
+// available.
+//
+// waitForClusterResourceSetToApplyResources wait until all ClusterResourceSet resources are created in the matching
+// cluster.
+func waitForClusterResourceSetToApplyResources(
+	ctx context.Context,
+	input framework.WaitForClusterResourceSetToApplyResourcesInput,
+	intervals ...interface{},
+) {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForClusterResourceSetToApplyResources")
+	Expect(
+		input.ClusterProxy,
+	).ToNot(
+		BeNil(),
+		"Invalid argument. input.ClusterProxy can't be nil when calling WaitForClusterResourceSetToApplyResources",
+	)
+	Expect(
+		input.Cluster,
+	).ToNot(
+		BeNil(),
+		"Invalid argument. input.Cluster can't be nil when calling WaitForClusterResourceSetToApplyResources",
+	)
+	Expect(
+		input.ClusterResourceSet,
+	).NotTo(
+		BeNil(),
+		"Invalid argument. input.ClusterResourceSet can't be nil when calling WaitForClusterResourceSetToApplyResources",
+	)
+
+	fmt.Fprintln(GinkgoWriter, "Waiting until the binding is created for the workload cluster")
+	Eventually(func() bool {
+		binding := &addonsv1.ClusterResourceSetBinding{}
+		err := input.ClusterProxy.GetClient().
+			Get(ctx, types.NamespacedName{Name: input.Cluster.Name, Namespace: input.Cluster.Namespace}, binding)
+		return err == nil
+	}, intervals...).Should(BeTrue())
+
+	fmt.Fprintln(GinkgoWriter, "Waiting until the resource is created in the workload cluster")
+	Eventually(func() bool {
+		binding := &addonsv1.ClusterResourceSetBinding{}
+		Expect(
+			input.ClusterProxy.GetClient().
+				Get(ctx, types.NamespacedName{Name: input.Cluster.Name, Namespace: input.Cluster.Namespace}, binding),
+		).To(Succeed())
+
+		for _, resource := range input.ClusterResourceSet.Spec.Resources {
+			var configSource client.Object
+
+			switch resource.Kind {
+			case string(addonsv1.SecretClusterResourceSetResourceKind):
+				configSource = &corev1.Secret{}
+			case string(addonsv1.ConfigMapClusterResourceSetResourceKind):
+				configSource = &corev1.ConfigMap{}
+			}
+
+			if err := input.ClusterProxy.GetClient().Get(
+				ctx, types.NamespacedName{Name: resource.Name, Namespace: input.ClusterResourceSet.Namespace}, configSource,
+			); err != nil {
+				// If the resource is missing, CRS will not requeue but retry at each reconcile,
+				// because this is not an error. So, we are only interested in seeing the resources that exist to be applied by CRS.
+				continue
+			}
+
+			// Fix a bug from upstream to check all bindings.
+			resourceApplied := false
+			for _, binding := range binding.Spec.Bindings {
+				if binding.IsApplied(resource) {
+					resourceApplied = true
+					break
+				}
+			}
+			if !resourceApplied {
+				return false
+			}
+		}
+		return true
+	}, intervals...).Should(BeTrue())
+}

test/e2e/cni_helpers.go

@@ -18,34 +18,6 @@ import (
 	"github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1"
 )
 
-type WaitForAddonsToBeReadyInWorkloadClusterInput struct {
-	AddonsConfig                v1alpha1.Addons
-	WorkloadCluster             *capiv1.Cluster
-	ClusterProxy                framework.ClusterProxy
-	DeploymentIntervals         []interface{}
-	DaemonSetIntervals          []interface{}
-	HelmReleaseIntervals        []interface{}
-	ClusterResourceSetIntervals []interface{}
-}
-
-func WaitForAddonsToBeReadyInWorkloadCluster(
-	ctx context.Context,
-	input WaitForAddonsToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
-) {
-	WaitForCNIToBeReadyInWorkloadCluster(
-		ctx,
-		WaitForCNIToBeReadyInWorkloadClusterInput{
-			CNI:                         input.AddonsConfig.CNI,
-			WorkloadCluster:             input.WorkloadCluster,
-			ClusterProxy:                input.ClusterProxy,
-			DeploymentIntervals:         input.DeploymentIntervals,
-			DaemonSetIntervals:          input.DaemonSetIntervals,
-			HelmReleaseIntervals:        input.HelmReleaseIntervals,
-			ClusterResourceSetIntervals: input.ClusterResourceSetIntervals,
-		},
-	)
-}
-
 type WaitForCNIToBeReadyInWorkloadClusterInput struct {
 	CNI                         *v1alpha1.CNI
 	WorkloadCluster             *capiv1.Cluster

test/e2e/nfd_helpers.go

@@ -0,0 +1,80 @@
+//go:build e2e
+
+// Copyright 2024 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/test/framework"
+
+	"github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1"
+)
+
+type WaitForNFDToBeReadyInWorkloadClusterInput struct {
+	NFD                         *v1alpha1.NFD
+	WorkloadCluster             *capiv1.Cluster
+	ClusterProxy                framework.ClusterProxy
+	DeploymentIntervals         []interface{}
+	DaemonSetIntervals          []interface{}
+	HelmReleaseIntervals        []interface{}
+	ClusterResourceSetIntervals []interface{}
+}
+
+func WaitForNFDToBeReadyInWorkloadCluster(
+	ctx context.Context,
+	input WaitForNFDToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
+) {
+	if input.NFD == nil {
+		return
+	}
+
+	waitForClusterResourceSetToApplyResourcesInCluster(
+		ctx,
+		waitForClusterResourceSetToApplyResourcesInClusterInput{
+			name:         "node-feature-discovery-" + input.WorkloadCluster.Name,
+			clusterProxy: input.ClusterProxy,
+			cluster:      input.WorkloadCluster,
+			intervals:    input.ClusterResourceSetIntervals,
+		},
+	)
+
+	workloadClusterClient := input.ClusterProxy.GetWorkloadCluster(
+		ctx, input.WorkloadCluster.Namespace, input.WorkloadCluster.Name,
+	).GetClient()
+
+	WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
+		Getter: workloadClusterClient,
+		Deployment: &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "node-feature-discovery-gc",
+				Namespace: "node-feature-discovery",
+			},
+		},
+	}, input.DeploymentIntervals...)
+
+	WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
+		Getter: workloadClusterClient,
+		Deployment: &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "node-feature-discovery-master",
+				Namespace: "node-feature-discovery",
+			},
+		},
+	}, input.DeploymentIntervals...)
+
+	WaitForDaemonSetsAvailable(ctx, WaitForDaemonSetsAvailableInput{
+		Getter: workloadClusterClient,
+		DaemonSet: &appsv1.DaemonSet{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "node-feature-discovery-worker",
+				Namespace: "node-feature-discovery",
+			},
+		},
+	}, input.DaemonSetIntervals...)
+}