From 8d965549a61b8e29bcb87061733d519b7e9cab23 Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova Date: Mon, 23 Dec 2024 18:01:11 +0400 Subject: [PATCH 1/2] Added support to enable custom addons in EKS template And enable CSI driver addon by default --- templates/cluster/aws-eks/Chart.yaml | 2 +- .../templates/awsmanagedcontrolplane.yaml | 4 +++ templates/cluster/aws-eks/values.schema.json | 33 +++++++++++++++++++ templates/cluster/aws-eks/values.yaml | 8 +++++ ...{aws-eks-0-0-2.yaml => aws-eks-0-0-3.yaml} | 4 +-- 5 files changed, 48 insertions(+), 3 deletions(-) rename templates/provider/hmc-templates/files/templates/{aws-eks-0-0-2.yaml => aws-eks-0-0-3.yaml} (86%) diff --git a/templates/cluster/aws-eks/Chart.yaml b/templates/cluster/aws-eks/Chart.yaml index f0707095d..b8b215478 100644 --- a/templates/cluster/aws-eks/Chart.yaml +++ b/templates/cluster/aws-eks/Chart.yaml @@ -6,7 +6,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.2 +version: 0.0.3 annotations: cluster.x-k8s.io/provider: infrastructure-aws cluster.x-k8s.io/infrastructure-aws: v1beta2 diff --git a/templates/cluster/aws-eks/templates/awsmanagedcontrolplane.yaml b/templates/cluster/aws-eks/templates/awsmanagedcontrolplane.yaml index 6807e5033..c9541ec4d 100644 --- a/templates/cluster/aws-eks/templates/awsmanagedcontrolplane.yaml +++ b/templates/cluster/aws-eks/templates/awsmanagedcontrolplane.yaml @@ -12,3 +12,7 @@ spec: identityRef: kind: {{ .Values.clusterIdentity.kind }} name: {{ .Values.clusterIdentity.name }} + {{- with .Values.addons }} + addons: + {{- toYaml . | nindent 4 }} + {{- end }} diff --git a/templates/cluster/aws-eks/values.schema.json b/templates/cluster/aws-eks/values.schema.json index 756857410..bd2adb626 100644 --- a/templates/cluster/aws-eks/values.schema.json +++ b/templates/cluster/aws-eks/values.schema.json @@ -129,6 +129,39 @@ } } }, + "addons": { + "description": "The EKS addons to enable with the EKS cluster", + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "description": "The name of the addon", + "type": "string" + }, + "version": { + "description": "The version of the addon to use", + "type": "string" + }, + "configuration": { + "description": "Optional configuration of the EKS addon in YAML format", + "type": "string" + }, + "conflictResolution": { + "description": "ConflictResolution is used to declare what should happen if there are parameter conflicts. Defaults to none", + "type": "string" + }, + "serviceAccountRoleARN": { + "description": "ServiceAccountRoleArn is the ARN of an IAM role to bind to the addons service account", + "type": "string" + } + } + } + }, "kubernetes": { "description": "Kubernetes parameters", "type": "object", diff --git a/templates/cluster/aws-eks/values.yaml b/templates/cluster/aws-eks/values.yaml index 188b7818b..afe8c5053 100644 --- a/templates/cluster/aws-eks/values.yaml +++ b/templates/cluster/aws-eks/values.yaml @@ -1,3 +1,4 @@ + # Cluster parameters workersNumber: 1 @@ -29,6 +30,13 @@ worker: org: "" baseOS: "" +addons: +- name: aws-ebs-csi-driver + version: v1.37.0-eksbuild.1 + configuration: | + defaultStorageClass: + enabled: true + # Kubernetes version kubernetes: version: v1.30.4 diff --git a/templates/provider/hmc-templates/files/templates/aws-eks-0-0-2.yaml b/templates/provider/hmc-templates/files/templates/aws-eks-0-0-3.yaml similarity index 86% rename from templates/provider/hmc-templates/files/templates/aws-eks-0-0-2.yaml rename to templates/provider/hmc-templates/files/templates/aws-eks-0-0-3.yaml index fa3d75fd7..1ca8668d6 100644 --- a/templates/provider/hmc-templates/files/templates/aws-eks-0-0-2.yaml +++ b/templates/provider/hmc-templates/files/templates/aws-eks-0-0-3.yaml @@ -1,14 +1,14 @@ apiVersion: hmc.mirantis.com/v1alpha1 kind: ClusterTemplate metadata: - name: aws-eks-0-0-2 + name: aws-eks-0-0-3 annotations: helm.sh/resource-policy: keep spec: helm: chartSpec: chart: aws-eks - version: 0.0.2 + version: 0.0.3 interval: 10m0s sourceRef: kind: HelmRepository From a0b4ec87cbfa037efeab47fd5fe4a104a21a8bb4 Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova Date: Mon, 23 Dec 2024 19:48:24 +0400 Subject: [PATCH 2/2] Add EKS e2e testing --- config/dev/eks-clusterdeployment.yaml | 2 +- .../clusterdeployment/clusterdeployment.go | 6 ++ .../clusterdeployment/providervalidator.go | 20 +++++- .../resources/aws-eks.yaml.tpl | 13 ++++ .../e2e/clusterdeployment/validate_deleted.go | 40 ++++++------ .../clusterdeployment/validate_deployed.go | 61 ++++++++++++------- test/e2e/kubeclient/kubeclient.go | 12 ++++ test/e2e/provider_aws_test.go | 33 ++++++++++ 8 files changed, 143 insertions(+), 44 deletions(-) create mode 100644 test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl diff --git a/config/dev/eks-clusterdeployment.yaml b/config/dev/eks-clusterdeployment.yaml index 8973e7ffd..3735e286f 100644 --- a/config/dev/eks-clusterdeployment.yaml +++ b/config/dev/eks-clusterdeployment.yaml @@ -4,7 +4,7 @@ metadata: name: eks-dev namespace: ${NAMESPACE} spec: - template: aws-eks-0-0-2 + template: aws-eks-0-0-3 credential: "aws-cluster-identity-cred" config: region: ${AWS_REGION} diff --git a/test/e2e/clusterdeployment/clusterdeployment.go b/test/e2e/clusterdeployment/clusterdeployment.go index 4492f4dc0..0d2417356 100644 --- a/test/e2e/clusterdeployment/clusterdeployment.go +++ b/test/e2e/clusterdeployment/clusterdeployment.go @@ -46,6 +46,7 @@ type Template string const ( TemplateAWSStandaloneCP Template = "aws-standalone-cp" TemplateAWSHostedCP Template = "aws-hosted-cp" + TemplateAWSEKS Template = "aws-eks" TemplateAzureHostedCP Template = "azure-hosted-cp" TemplateAzureStandaloneCP Template = "azure-standalone-cp" TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp" @@ -58,6 +59,9 @@ var awsStandaloneCPClusterDeploymentTemplateBytes []byte //go:embed resources/aws-hosted-cp.yaml.tpl var awsHostedCPClusterDeploymentTemplateBytes []byte +//go:embed resources/aws-eks.yaml.tpl +var awsEksClusterDeploymentTemplateBytes []byte + //go:embed resources/azure-standalone-cp.yaml.tpl var azureStandaloneCPClusterDeploymentTemplateBytes []byte @@ -126,6 +130,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { EnvVarAWSSecurityGroupID, }) clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes + case TemplateAWSEKS: + clusterDeploymentTemplateBytes = awsEksClusterDeploymentTemplateBytes case TemplateVSphereStandaloneCP: clusterDeploymentTemplateBytes = vsphereStandaloneCPClusterDeploymentTemplateBytes case TemplateVSphereHostedCP: diff --git a/test/e2e/clusterdeployment/providervalidator.go b/test/e2e/clusterdeployment/providervalidator.go index 7ca81d1d2..552ac4237 100644 --- a/test/e2e/clusterdeployment/providervalidator.go +++ b/test/e2e/clusterdeployment/providervalidator.go @@ -65,6 +65,15 @@ func NewProviderValidator(template Template, clusterName string, action Validati case TemplateAWSStandaloneCP, TemplateAWSHostedCP: resourcesToValidate["ccm"] = validateCCM resourceOrder = append(resourceOrder, "ccm") + case TemplateAWSEKS: + resourcesToValidate = map[string]resourceValidationFunc{ + "clusters": validateCluster, + "machines": validateMachines, + "aws-managed-control-planes": validateAWSManagedControlPlanes, + "csi-driver": validateCSIDriver, + "ccm": validateCCM, + } + resourceOrder = []string{"clusters", "machines", "aws-managed-control-planes", "csi-driver", "ccm"} case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") } @@ -72,9 +81,16 @@ func NewProviderValidator(template Template, clusterName string, action Validati resourcesToValidate = map[string]resourceValidationFunc{ "clusters": validateClusterDeleted, "machinedeployments": validateMachineDeploymentsDeleted, - "control-planes": validateK0sControlPlanesDeleted, } - resourceOrder = []string{"clusters", "machinedeployments", "control-planes"} + resourceOrder = []string{"clusters", "machinedeployments"} + switch template { + case TemplateAWSEKS: + resourcesToValidate["aws-managed-control-planes"] = validateAWSManagedControlPlanesDeleted + resourceOrder = append(resourceOrder, "aws-managed-control-planes") + default: + resourcesToValidate["control-planes"] = validateK0sControlPlanesDeleted + resourceOrder = append(resourceOrder, "control-planes") + } } return &ProviderValidator{ diff --git a/test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl b/test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl new file mode 100644 index 000000000..9e54499dc --- /dev/null +++ b/test/e2e/clusterdeployment/resources/aws-eks.yaml.tpl @@ -0,0 +1,13 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: ClusterDeployment +metadata: + name: ${CLUSTER_DEPLOYMENT_NAME} +spec: + template: aws-eks-0-0-3 + credential: ${AWS_CLUSTER_IDENTITY}-cred + config: + region: ${AWS_REGION} + workersNumber: ${WORKERS_NUMBER:=1} + publicIP: ${AWS_PUBLIC_IP:=true} + worker: + instanceType: ${AWS_INSTANCE_TYPE:=t3.small} diff --git a/test/e2e/clusterdeployment/validate_deleted.go b/test/e2e/clusterdeployment/validate_deleted.go index 1789ec879..55c1fa97d 100644 --- a/test/e2e/clusterdeployment/validate_deleted.go +++ b/test/e2e/clusterdeployment/validate_deleted.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -69,17 +70,7 @@ func validateMachineDeploymentsDeleted(ctx context.Context, kc *kubeclient.KubeC if err != nil && !apierrors.IsNotFound(err) { return err } - - var mdNames []string - if len(machineDeployments) > 0 { - for _, md := range machineDeployments { - mdNames = append(mdNames, md.GetName()) - - return fmt.Errorf("machine deployments still exist: %s", mdNames) - } - } - - return nil + return validateObjectsRemoved("MachineDeployments", machineDeployments) } // validateK0sControlPlanesDeleted validates that all k0scontrolplanes have @@ -89,15 +80,26 @@ func validateK0sControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeCli if err != nil && !apierrors.IsNotFound(err) { return err } + return validateObjectsRemoved("K0sControlPlanes", controlPlanes) +} - var cpNames []string - if len(controlPlanes) > 0 { - for _, cp := range controlPlanes { - cpNames = append(cpNames, cp.GetName()) - - return fmt.Errorf("k0s control planes still exist: %s", cpNames) - } +// validateAWSManagedControlPlanesDeleted validates that all AWSManagedControlPlanes have +// been deleted. +func validateAWSManagedControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName) + if err != nil && !apierrors.IsNotFound(err) { + return err } + return validateObjectsRemoved("AWSManagedControlPlane", controlPlanes) +} - return nil +func validateObjectsRemoved(kind string, objs []unstructured.Unstructured) error { + if len(objs) == 0 { + return nil + } + names := make([]string, len(objs)) + for _, cp := range objs { + names = append(names, cp.GetName()) + } + return fmt.Errorf("one or more %s still exist: %s", kind, strings.Join(names, ", ")) } diff --git a/test/e2e/clusterdeployment/validate_deployed.go b/test/e2e/clusterdeployment/validate_deployed.go index 7f750ecc1..6c6ad0f2f 100644 --- a/test/e2e/clusterdeployment/validate_deployed.go +++ b/test/e2e/clusterdeployment/validate_deployed.go @@ -16,6 +16,7 @@ package clusterdeployment import ( "context" + "errors" "fmt" "strings" @@ -27,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" - "github.com/Mirantis/hmc/internal/utils/status" "github.com/Mirantis/hmc/test/e2e/kubeclient" "github.com/Mirantis/hmc/test/utils" ) @@ -104,38 +104,55 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl return err } + var errs error for _, controlPlane := range controlPlanes { if err := utils.ValidateObjectNamePrefix(&controlPlane, clusterName); err != nil { - Fail(err.Error()) + errs = errors.Join(errs, err) + continue } - objKind, objName := status.ObjKindName(&controlPlane) - // k0s does not use the metav1.Condition type for status.conditions, // instead it uses a custom type so we can't use // ValidateConditionsTrue here, instead we'll check for "ready: true". - objStatus, found, err := unstructured.NestedFieldCopy(controlPlane.Object, "status") - if !found { - return fmt.Errorf("no status found for %s: %s", objKind, objName) - } - if err != nil { - return fmt.Errorf("failed to get status conditions for %s: %s: %w", objKind, objName, err) - } - - st, ok := objStatus.(map[string]any) - if !ok { - return fmt.Errorf("expected K0sControlPlane condition to be type map[string]any, got: %T", objStatus) - } + errs = errors.Join(errs, validateReadyStatus(controlPlane)) + } - if _, ok := st["ready"]; !ok { - return fmt.Errorf("%s %s has no 'ready' status", objKind, objName) - } + return errs +} - if v, ok := st["ready"].(bool); !ok || !v { - return fmt.Errorf("%s %s is not ready, status: %+v", objKind, objName, st) - } +func validateAWSManagedControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName) + if err != nil { + return err + } + var errs error + for _, controlPlane := range controlPlanes { + errs = errors.Join(errs, validateReadyStatus(controlPlane)) } + return errs +} +// validateReadyStatus validates if the provided object has ready status +func validateReadyStatus(obj unstructured.Unstructured) error { + name := obj.GetName() + kind := obj.GetKind() + objStatus, found, err := unstructured.NestedFieldCopy(obj.Object, "status") + if !found { + return fmt.Errorf("no status found for %s: %s", kind, name) + } + if err != nil { + return fmt.Errorf("failed to get status conditions for %s: %s: %w", kind, name, err) + } + st, ok := objStatus.(map[string]any) + if !ok { + return fmt.Errorf("expected %s condition to be type map[string]any, got: %T", kind, objStatus) + } + if _, ok := st["ready"]; !ok { + return fmt.Errorf("%s %s has no 'ready' status", kind, name) + } + if v, ok := st["ready"].(bool); !ok || !v { + return fmt.Errorf("%s %s is not ready, status: %+v", kind, name, st) + } return nil } diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index 45e8fdb97..e5ad2b200 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -279,3 +279,15 @@ func (kc *KubeClient) ListK0sControlPlanes( Resource: "k0scontrolplanes", }, clusterName) } + +func (kc *KubeClient) ListAWSManagedControlPlanes( + ctx context.Context, clusterName string, +) ([]unstructured.Unstructured, error) { + GinkgoHelper() + + return kc.listResource(ctx, schema.GroupVersionResource{ + Group: "controlplane.cluster.x-k8s.io", + Version: "v1beta2", + Resource: "awsmanagedcontrolplanes", + }, clusterName) +} diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 276689032..a8c7588be 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -186,4 +186,37 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order time.Second).Should(Succeed()) */ }) + + It("should work with an AWS provider (EKS)", func() { + templateBy(clusterdeployment.TemplateAWSEKS, "creating a ClusterDeployment") + cd := clusterdeployment.GetUnstructured(clusterdeployment.TemplateAWSEKS) + clusterName = cd.GetName() + + eksDeleteFunc := kc.CreateClusterDeployment(context.Background(), cd) + + templateBy(clusterdeployment.TemplateAWSEKS, "waiting for infrastructure to deploy successfully") + deploymentValidator := clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSEKS, + clusterName, + clusterdeployment.ValidationActionDeploy, + ) + + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }, 30*time.Minute, 10*time.Second).Should(Succeed()) + + if !noCleanup() { + templateBy(clusterdeployment.TemplateAWSEKS, "deleting the ClusterDeployment") + Expect(eksDeleteFunc()).NotTo(HaveOccurred()) + + deletionValidator := clusterdeployment.NewProviderValidator( + clusterdeployment.TemplateAWSEKS, + clusterName, + clusterdeployment.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }, 20*time.Minute, 10*time.Second).Should(Succeed()) + } + }) })