From 9e202888e50da426ea9ca5f38a7c4380ece66728 Mon Sep 17 00:00:00 2001 From: Carlos Salas Date: Thu, 10 Oct 2024 11:48:48 +0200 Subject: [PATCH 1/3] test: cover azure provisioning via cluster class Signed-off-by: Carlos Salas --- test/e2e/const.go | 3 + .../cluster-templates/azure-aks-topology.yaml | 150 ++++++++++++++++++ test/e2e/specs/import_gitops_mgmtv3.go | 1 + .../import-gitops-v3/import_gitops_v3_test.go | 31 ++++ 4 files changed, 185 insertions(+) create mode 100644 test/e2e/data/cluster-templates/azure-aks-topology.yaml diff --git a/test/e2e/const.go b/test/e2e/const.go index f238cf11..366d3622 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -78,6 +78,9 @@ var ( //go:embed data/cluster-templates/azure-aks-mmp.yaml CAPIAzureAKSMMP []byte + //go:embed data/cluster-templates/azure-aks-topology.yaml + CAPIAzureAKSTopology []byte + //go:embed data/cluster-templates/vsphere-kubeadm.yaml CAPIvSphereKubeadm []byte diff --git a/test/e2e/data/cluster-templates/azure-aks-topology.yaml b/test/e2e/data/cluster-templates/azure-aks-topology.yaml new file mode 100644 index 00000000..523faafe --- /dev/null +++ b/test/e2e/data/cluster-templates/azure-aks-topology.yaml @@ -0,0 +1,150 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: ${CLUSTER_CLASS_NAME} + namespace: default +spec: + controlPlane: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME} + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: cluster-identity + location: southcentralus + resourceGroupName: "${CLUSTER_NAME}" + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + mode: System + name: pool0 + sku: Standard_D2s_v3 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + mode: User + name: pool1 + sku: Standard_D2s_v3 +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: {} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: cluster-identity + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: cluster-identity-secret + namespace: default + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + version: ${KUBERNETES_VERSION} + workers: + machinePools: + - class: default-system + name: mp-0 + replicas: 1 + - class: default-worker + name: mp-1 + replicas: 1 diff --git a/test/e2e/specs/import_gitops_mgmtv3.go b/test/e2e/specs/import_gitops_mgmtv3.go index e25f2877..a12ead5a 100644 --- a/test/e2e/specs/import_gitops_mgmtv3.go +++ b/test/e2e/specs/import_gitops_mgmtv3.go @@ -229,6 +229,7 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM additionalVars := map[string]string{ "CLUSTER_NAME": input.ClusterName, + "CLUSTER_CLASS_NAME": fmt.Sprintf("%s-class", input.ClusterName), "WORKER_MACHINE_COUNT": strconv.Itoa(workerMachineCount), "CONTROL_PLANE_MACHINE_COUNT": strconv.Itoa(controlPlaneMachineCount), } diff --git a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go index 1575a35e..81bb0a0b 100644 --- a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go +++ b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go @@ -95,6 +95,37 @@ var _ = Describe("[Docker] [RKE2] - [management.cattle.io/v3] Create and delete }) }) +var _ = Describe("[Azure] [AKS] - [management.cattle.io/v3] Create and delete CAPI cluster from cluster class", Label(e2e.FullTestLabel), func() { + BeforeEach(func() { + komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) + komega.SetContext(ctx) + }) + + specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput { + return specs.CreateMgmtV3UsingGitOpsSpecInput{ + E2EConfig: e2eConfig, + BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, + ClusterctlConfigPath: flagVals.ConfigPath, + ArtifactFolder: artifactsFolder, + ClusterTemplate: e2e.CAPIAzureAKSTopology, + ClusterName: "highlander-e2e-topology", + ControlPlaneMachineCount: ptr.To[int](1), + WorkerMachineCount: ptr.To[int](1), + GitAddr: giteaResult.GitAddress, + GitAuthSecretName: e2e.AuthSecretName, + SkipCleanup: false, + SkipDeletionTest: false, + LabelNamespace: true, + RancherServerURL: hostName, + CAPIClusterCreateWaitName: "wait-capz-create-cluster", + DeleteClusterWaitName: "wait-aks-delete", + CapiClusterOwnerLabel: e2e.CapiClusterOwnerLabel, + CapiClusterOwnerNamespaceLabel: e2e.CapiClusterOwnerNamespaceLabel, + OwnedLabelName: e2e.OwnedLabelName, + } + }) +}) + var _ = Describe("[Azure] [AKS] - [management.cattle.io/v3] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() { BeforeEach(func() { komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) From 02f51f3d1d825aa2ea5e4206b94d038412a9627e Mon Sep 17 00:00:00 2001 From: Carlos Salas Date: Thu, 10 Oct 2024 12:01:51 +0200 Subject: [PATCH 2/3] test: remove "standard" aks e2e scenario Signed-off-by: Carlos Salas --- test/e2e/const.go | 3 - .../data/cluster-templates/azure-aks-mmp.yaml | 131 ------------------ .../import-gitops-v3/import_gitops_v3_test.go | 31 ----- 3 files changed, 165 deletions(-) delete mode 100644 test/e2e/data/cluster-templates/azure-aks-mmp.yaml diff --git a/test/e2e/const.go b/test/e2e/const.go index 366d3622..b476cc13 100644 --- a/test/e2e/const.go +++ b/test/e2e/const.go @@ -75,9 +75,6 @@ var ( //go:embed data/cluster-templates/aws-eks-mmp.yaml CAPIAwsEKSMMP []byte - //go:embed data/cluster-templates/azure-aks-mmp.yaml - CAPIAzureAKSMMP []byte - //go:embed data/cluster-templates/azure-aks-topology.yaml CAPIAzureAKSTopology []byte diff --git a/test/e2e/data/cluster-templates/azure-aks-mmp.yaml b/test/e2e/data/cluster-templates/azure-aks-mmp.yaml deleted file mode 100644 index df9f3065..00000000 --- a/test/e2e/data/cluster-templates/azure-aks-mmp.yaml +++ /dev/null @@ -1,131 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: "${CLUSTER_NAME}" - namespace: default -spec: - clusterNetwork: - services: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureManagedControlPlane - name: "${CLUSTER_NAME}" - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureManagedCluster - name: "${CLUSTER_NAME}" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedControlPlane -metadata: - annotations: - "helm.sh/resource-policy": keep - name: "${CLUSTER_NAME}" - namespace: default -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: cluster-identity - location: southcentralus - resourceGroupName: "${CLUSTER_NAME}" - nodeResourceGroupName: "${CLUSTER_NAME}-nodes" - sshPublicKey: "" - subscriptionID: "${AZURE_SUBSCRIPTION_ID}" - version: "${KUBERNETES_VERSION}" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedCluster -metadata: - annotations: - "helm.sh/resource-policy": keep - name: "${CLUSTER_NAME}" - namespace: default ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - annotations: - "helm.sh/resource-policy": keep - name: "${CLUSTER_NAME}-pool0" - namespace: default -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - template: - metadata: {} - spec: - bootstrap: - dataSecretName: "" - clusterName: "${CLUSTER_NAME}" - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureManagedMachinePool - name: "${CLUSTER_NAME}-pool0" - version: "${KUBERNETES_VERSION}" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - annotations: - "helm.sh/resource-policy": keep - name: "${CLUSTER_NAME}-pool0" - namespace: default -spec: - mode: System - name: pool0 - sku: Standard_D2s_v3 ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - annotations: - "helm.sh/resource-policy": keep - name: "${CLUSTER_NAME}-pool1" - namespace: default -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - metadata: {} - spec: - bootstrap: - dataSecretName: "" - clusterName: "${CLUSTER_NAME}" - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureManagedMachinePool - name: "${CLUSTER_NAME}-pool1" - version: "${KUBERNETES_VERSION}" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - annotations: - "helm.sh/resource-policy": keep - name: "${CLUSTER_NAME}-pool1" - namespace: default -spec: - mode: User - name: pool1 - sku: Standard_D2s_v3 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - annotations: - "helm.sh/resource-policy": keep - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: cluster-identity - namespace: default -spec: - allowedNamespaces: {} - clientID: "${AZURE_CLIENT_ID}" - clientSecret: - name: cluster-identity-secret - namespace: default - tenantID: "${AZURE_TENANT_ID}" - type: ServicePrincipal diff --git a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go index 81bb0a0b..c2ab3224 100644 --- a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go +++ b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go @@ -126,37 +126,6 @@ var _ = Describe("[Azure] [AKS] - [management.cattle.io/v3] Create and delete CA }) }) -var _ = Describe("[Azure] [AKS] - [management.cattle.io/v3] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() { - BeforeEach(func() { - komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) - komega.SetContext(ctx) - }) - - specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput { - return specs.CreateMgmtV3UsingGitOpsSpecInput{ - E2EConfig: e2eConfig, - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - ClusterctlConfigPath: flagVals.ConfigPath, - ArtifactFolder: artifactsFolder, - ClusterTemplate: e2e.CAPIAzureAKSMMP, - ClusterName: "highlander-e2e-clusterv3-2", - ControlPlaneMachineCount: ptr.To[int](1), - WorkerMachineCount: ptr.To[int](1), - GitAddr: giteaResult.GitAddress, - GitAuthSecretName: e2e.AuthSecretName, - SkipCleanup: false, - SkipDeletionTest: false, - LabelNamespace: true, - RancherServerURL: hostName, - CAPIClusterCreateWaitName: "wait-capz-create-cluster", - DeleteClusterWaitName: "wait-aks-delete", - CapiClusterOwnerLabel: e2e.CapiClusterOwnerLabel, - CapiClusterOwnerNamespaceLabel: e2e.CapiClusterOwnerNamespaceLabel, - OwnedLabelName: e2e.OwnedLabelName, - } - }) -}) - var _ = Describe("[AWS] [EKS] - [management.cattle.io/v3] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.FullTestLabel), func() { BeforeEach(func() { komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) From 9e7cd417e877c976dec7238e9998ed07469e1460 Mon Sep 17 00:00:00 2001 From: Carlos Salas Date: Mon, 14 Oct 2024 08:24:10 +0200 Subject: [PATCH 3/3] test: reference rke2 bootstrap in aks class template Signed-off-by: Carlos Salas --- test/e2e/data/cluster-templates/azure-aks-topology.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/data/cluster-templates/azure-aks-topology.yaml b/test/e2e/data/cluster-templates/azure-aks-topology.yaml index 523faafe..14be24b3 100644 --- a/test/e2e/data/cluster-templates/azure-aks-topology.yaml +++ b/test/e2e/data/cluster-templates/azure-aks-topology.yaml @@ -21,7 +21,7 @@ spec: bootstrap: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate + kind: RKE2ConfigTemplate name: ${CLUSTER_NAME}-pool0 infrastructure: ref: @@ -33,7 +33,7 @@ spec: bootstrap: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate + kind: RKE2ConfigTemplate name: ${CLUSTER_NAME}-pool1 infrastructure: ref: @@ -92,7 +92,7 @@ spec: sku: Standard_D2s_v3 --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate +kind: RKE2ConfigTemplate metadata: name: ${CLUSTER_NAME}-pool0 namespace: default @@ -101,7 +101,7 @@ spec: spec: {} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate +kind: RKE2ConfigTemplate metadata: name: ${CLUSTER_NAME}-pool1 namespace: default