Add EKS e2e testing

config/dev/eks-clusterdeployment.yaml

@@ -4,7 +4,7 @@ metadata:
   name: eks-dev
   namespace: ${NAMESPACE}
 spec:
-  template: aws-eks-0-0-2
+  template: aws-eks-0-0-3
   credential: "aws-cluster-identity-cred"
   config:
     region: ${AWS_REGION}

templates/cluster/aws-eks/Chart.yaml

@@ -6,7 +6,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.2
+version: 0.0.3
 annotations:
   cluster.x-k8s.io/provider: infrastructure-aws
   cluster.x-k8s.io/infrastructure-aws: v1beta2

templates/cluster/aws-eks/templates/awsmanagedcontrolplane.yaml

@@ -12,3 +12,7 @@ spec:
   identityRef:
     kind: {{ .Values.clusterIdentity.kind }}
     name: {{ .Values.clusterIdentity.name }}
+  {{- with .Values.addons }}
+  addons:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}

templates/cluster/aws-eks/values.schema.json

@@ -129,6 +129,39 @@
         }
       }
     },
+    "addons": {
+      "description": "The EKS addons to enable with the EKS cluster",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "name",
+          "version"
+        ],
+        "properties": {
+          "name": {
+            "description": "The name of the addon",
+            "type": "string"
+          },
+          "version": {
+            "description": "The version of the addon to use",
+            "type": "string"
+          },
+          "configuration": {
+            "description": "Optional configuration of the EKS addon in YAML format",
+            "type": "string"
+          },
+          "conflictResolution": {
+            "description": "ConflictResolution is used to declare what should happen if there are parameter conflicts. Defaults to none",
+            "type": "string"
+          },
+          "serviceAccountRoleARN": {
+            "description": "ServiceAccountRoleArn is the ARN of an IAM role to bind to the addons service account",
+            "type": "string"
+          }
+        }
+      }
+    },
     "kubernetes": {
       "description": "Kubernetes parameters",
       "type": "object",

templates/cluster/aws-eks/values.yaml

@@ -1,3 +1,4 @@
+
 # Cluster parameters
 workersNumber: 1
 
@@ -29,6 +30,13 @@ worker:
     org: ""
     baseOS: ""
 
+addons:
+- name: aws-ebs-csi-driver
+  version: v1.37.0-eksbuild.1
+  configuration: |
+    defaultStorageClass:
+      enabled: true
+
 # Kubernetes version
 kubernetes:
   version: v1.30.4

templates/provider/hmc-templates/files/templates/aws-eks-0-0-2.yaml → templates/provider/hmc-templates/files/templates/aws-eks-0-0-3.yaml

@@ -1,14 +1,14 @@
 apiVersion: hmc.mirantis.com/v1alpha1
 kind: ClusterTemplate
 metadata:
-  name: aws-eks-0-0-2
+  name: aws-eks-0-0-3
   annotations:
     helm.sh/resource-policy: keep
 spec:
   helm:
     chartSpec:
       chart: aws-eks
-      version: 0.0.2
+      version: 0.0.3
       interval: 10m0s
       sourceRef:
         kind: HelmRepository

test/e2e/clusterdeployment/clusterdeployment.go

@@ -46,6 +46,7 @@ type Template string
 const (
 	TemplateAWSStandaloneCP     Template = "aws-standalone-cp"
 	TemplateAWSHostedCP         Template = "aws-hosted-cp"
+	TemplateAWSEKS              Template = "aws-eks"
 	TemplateAzureHostedCP       Template = "azure-hosted-cp"
 	TemplateAzureStandaloneCP   Template = "azure-standalone-cp"
 	TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp"
@@ -58,6 +59,9 @@ var awsStandaloneCPClusterDeploymentTemplateBytes []byte
 //go:embed resources/aws-hosted-cp.yaml.tpl
 var awsHostedCPClusterDeploymentTemplateBytes []byte
 
+//go:embed resources/aws-eks.yaml.tpl
+var awsEksClusterDeploymentTemplateBytes []byte
+
 //go:embed resources/azure-standalone-cp.yaml.tpl
 var azureStandaloneCPClusterDeploymentTemplateBytes []byte
 
@@ -126,6 +130,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 			EnvVarAWSSecurityGroupID,
 		})
 		clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes
+	case TemplateAWSEKS:
+		clusterDeploymentTemplateBytes = awsEksClusterDeploymentTemplateBytes
 	case TemplateVSphereStandaloneCP:
 		clusterDeploymentTemplateBytes = vsphereStandaloneCPClusterDeploymentTemplateBytes
 	case TemplateVSphereHostedCP:

test/e2e/clusterdeployment/providervalidator.go

@@ -65,16 +65,32 @@ func NewProviderValidator(template Template, clusterName string, action Validati
 		case TemplateAWSStandaloneCP, TemplateAWSHostedCP:
 			resourcesToValidate["ccm"] = validateCCM
 			resourceOrder = append(resourceOrder, "ccm")
+		case TemplateAWSEKS:
+			resourcesToValidate = map[string]resourceValidationFunc{
+				"clusters":                   validateCluster,
+				"machines":                   validateMachines,
+				"aws-managed-control-planes": validateAWSManagedControlPlanes,
+				"csi-driver":                 validateCSIDriver,
+				"ccm":                        validateCCM,
+			}
+			resourceOrder = []string{"clusters", "machines", "aws-managed-control-planes", "csi-driver", "ccm"}
 		case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP:
 			delete(resourcesToValidate, "csi-driver")
 		}
 	} else {
 		resourcesToValidate = map[string]resourceValidationFunc{
 			"clusters":           validateClusterDeleted,
 			"machinedeployments": validateMachineDeploymentsDeleted,
-			"control-planes":     validateK0sControlPlanesDeleted,
 		}
-		resourceOrder = []string{"clusters", "machinedeployments", "control-planes"}
+		resourceOrder = []string{"clusters", "machinedeployments"}
+		switch template {
+		case TemplateAWSEKS:
+			resourcesToValidate["aws-managed-control-planes"] = validateAWSManagedControlPlanesDeleted
+			resourceOrder = append(resourceOrder, "aws-managed-control-planes")
+		default:
+			resourcesToValidate["control-planes"] = validateK0sControlPlanesDeleted
+			resourceOrder = append(resourceOrder, "control-planes")
+		}
 	}
 
 	return &ProviderValidator{

test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl

@@ -0,0 +1,13 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: ClusterDeployment
+metadata:
+  name: ${CLUSTER_DEPLOYMENT_NAME}
+spec:
+  template: aws-eks-0-0-3
+  credential: ${AWS_CLUSTER_IDENTITY}-cred
+  config:
+    region: ${AWS_REGION}
+    workersNumber: ${WORKERS_NUMBER:=1}
+    publicIP: ${AWS_PUBLIC_IP:=true}
+    worker:
+      instanceType: ${AWS_INSTANCE_TYPE:=t3.small}

test/e2e/clusterdeployment/validate_deleted.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -69,17 +70,7 @@ func validateMachineDeploymentsDeleted(ctx context.Context, kc *kubeclient.KubeC
 	if err != nil && !apierrors.IsNotFound(err) {
 		return err
 	}
-
-	var mdNames []string
-	if len(machineDeployments) > 0 {
-		for _, md := range machineDeployments {
-			mdNames = append(mdNames, md.GetName())
-
-			return fmt.Errorf("machine deployments still exist: %s", mdNames)
-		}
-	}
-
-	return nil
+	return validateObjectsRemoved("MachineDeployments", machineDeployments)
 }
 
 // validateK0sControlPlanesDeleted validates that all k0scontrolplanes have
@@ -89,15 +80,26 @@ func validateK0sControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeCli
 	if err != nil && !apierrors.IsNotFound(err) {
 		return err
 	}
+	return validateObjectsRemoved("K0sControlPlanes", controlPlanes)
+}
 
-	var cpNames []string
-	if len(controlPlanes) > 0 {
-		for _, cp := range controlPlanes {
-			cpNames = append(cpNames, cp.GetName())
-
-			return fmt.Errorf("k0s control planes still exist: %s", cpNames)
-		}
+// validateAWSManagedControlPlanesDeleted validates that all AWSManagedControlPlanes have
+// been deleted.
+func validateAWSManagedControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
+	controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return err
 	}
+	return validateObjectsRemoved("AWSManagedControlPlane", controlPlanes)
+}
 
-	return nil
+func validateObjectsRemoved(kind string, objs []unstructured.Unstructured) error {
+	if len(objs) == 0 {
+		return nil
+	}
+	names := make([]string, len(objs))
+	for _, cp := range objs {
+		names = append(names, cp.GetName())
+	}
+	return fmt.Errorf("one or more %s still exist: %s", kind, strings.Join(names, ", "))
 }

test/e2e/clusterdeployment/validate_deployed.go

@@ -16,6 +16,7 @@ package clusterdeployment
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 
@@ -27,7 +28,6 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
-	"github.com/Mirantis/hmc/internal/utils/status"
 	"github.com/Mirantis/hmc/test/e2e/kubeclient"
 	"github.com/Mirantis/hmc/test/utils"
 )
@@ -104,38 +104,55 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl
 		return err
 	}
 
+	var errs error
 	for _, controlPlane := range controlPlanes {
 		if err := utils.ValidateObjectNamePrefix(&controlPlane, clusterName); err != nil {
-			Fail(err.Error())
+			errs = errors.Join(errs, err)
+			continue
 		}
 
-		objKind, objName := status.ObjKindName(&controlPlane)
-
 		// k0s does not use the metav1.Condition type for status.conditions,
 		// instead it uses a custom type so we can't use
 		// ValidateConditionsTrue here, instead we'll check for "ready: true".
-		objStatus, found, err := unstructured.NestedFieldCopy(controlPlane.Object, "status")
-		if !found {
-			return fmt.Errorf("no status found for %s: %s", objKind, objName)
-		}
-		if err != nil {
-			return fmt.Errorf("failed to get status conditions for %s: %s: %w", objKind, objName, err)
-		}
-
-		st, ok := objStatus.(map[string]any)
-		if !ok {
-			return fmt.Errorf("expected K0sControlPlane condition to be type map[string]any, got: %T", objStatus)
-		}
+		errs = errors.Join(errs, validateReadyStatus(controlPlane))
+	}
 
-		if _, ok := st["ready"]; !ok {
-			return fmt.Errorf("%s %s has no 'ready' status", objKind, objName)
-		}
+	return errs
+}
 
-		if v, ok := st["ready"].(bool); !ok || !v {
-			return fmt.Errorf("%s %s is not ready, status: %+v", objKind, objName, st)
-		}
+func validateAWSManagedControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
+	controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName)
+	if err != nil {
+		return err
+	}
+	var errs error
+	for _, controlPlane := range controlPlanes {
+		errs = errors.Join(errs, validateReadyStatus(controlPlane))
 	}
+	return errs
+}
 
+// validateReadyStatus validates if the provided object has ready status
+func validateReadyStatus(obj unstructured.Unstructured) error {
+	name := obj.GetName()
+	kind := obj.GetKind()
+	objStatus, found, err := unstructured.NestedFieldCopy(obj.Object, "status")
+	if !found {
+		return fmt.Errorf("no status found for %s: %s", kind, name)
+	}
+	if err != nil {
+		return fmt.Errorf("failed to get status conditions for %s: %s: %w", kind, name, err)
+	}
+	st, ok := objStatus.(map[string]any)
+	if !ok {
+		return fmt.Errorf("expected %s condition to be type map[string]any, got: %T", kind, objStatus)
+	}
+	if _, ok := st["ready"]; !ok {
+		return fmt.Errorf("%s %s has no 'ready' status", kind, name)
+	}
+	if v, ok := st["ready"].(bool); !ok || !v {
+		return fmt.Errorf("%s %s is not ready, status: %+v", kind, name, st)
+	}
 	return nil
 }
 

test/e2e/kubeclient/kubeclient.go

@@ -279,3 +279,15 @@ func (kc *KubeClient) ListK0sControlPlanes(
 		Resource: "k0scontrolplanes",
 	}, clusterName)
 }
+
+func (kc *KubeClient) ListAWSManagedControlPlanes(
+	ctx context.Context, clusterName string,
+) ([]unstructured.Unstructured, error) {
+	GinkgoHelper()
+
+	return kc.listResource(ctx, schema.GroupVersionResource{
+		Group:    "controlplane.cluster.x-k8s.io",
+		Version:  "v1beta2",
+		Resource: "awsmanagedcontrolplanes",
+	}, clusterName)
+}

test/e2e/provider_aws_test.go

@@ -186,4 +186,37 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order
 				time.Second).Should(Succeed())
 		*/
 	})
+
+	It("should work with an AWS provider (EKS)", func() {
+		templateBy(clusterdeployment.TemplateAWSEKS, "creating a ClusterDeployment")
+		cd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAWSEKS)
+		clusterName = cd.GetName()
+
+		eksDeleteFunc := kc.CreateClusterDeployment(context.Background(), cd)
+
+		templateBy(clusterdeployment.TemplateAWSEKS, "waiting for infrastructure to deploy successfully")
+		deploymentValidator := clusterdeployment.NewProviderValidator(
+			clusterdeployment.TemplateAWSEKS,
+			clusterName,
+			clusterdeployment.ValidationActionDeploy,
+		)
+
+		Eventually(func() error {
+			return deploymentValidator.Validate(context.Background(), kc)
+		}, 30*time.Minute, 10*time.Second).Should(Succeed())
+
+		if !noCleanup() {
+			templateBy(clusterdeployment.TemplateAWSEKS, "deleting the ClusterDeployment")
+			Expect(eksDeleteFunc()).NotTo(HaveOccurred())
+
+			deletionValidator := clusterdeployment.NewProviderValidator(
+				clusterdeployment.TemplateAWSEKS,
+				clusterName,
+				clusterdeployment.ValidationActionDelete,
+			)
+			Eventually(func() error {
+				return deletionValidator.Validate(context.Background(), kc)
+			}, 20*time.Minute, 10*time.Second).Should(Succeed())
+		}
+	})
 })