test: add e2e for rancher with embedded capi disabled

Signed-off-by: Carlos Salas <carlos.salas@suse.com>

test/e2e/suites/embedded-capi-disabled/embedded_capi_disabled_test.go

@@ -0,0 +1,166 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2023 SUSE.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package embedded_capi_disabled
+
+import (
+	"fmt"
+	"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/drone/envsubst/v2"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
+
+	provisioningv1 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/provisioning/v1"
+	"github.com/rancher-sandbox/rancher-turtles/test/e2e"
+	turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework"
+)
+
+var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still work", Label(e2e.FullTestLabel), func() {
+
+	BeforeEach(func() {
+		komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
+		komega.SetContext(ctx)
+	})
+
+	It("Should create a RKE2 cluster in Azure", func() {
+		azSubId := e2eConfig.GetVariable(e2e.AzureSubIDVar)
+		Expect(azSubId).ToNot(BeEmpty(), "Azure Subscription ID is required")
+		azClientId := e2eConfig.GetVariable(e2e.AzureClientIDVar)
+		Expect(azSubId).ToNot(BeEmpty(), "Azure Client ID is required")
+		azClientSecret := e2eConfig.GetVariable(e2e.AzureClientSecretVar)
+		Expect(azSubId).ToNot(BeEmpty(), "Azure Client Secret is required")
+
+		rke2Version := e2eConfig.GetVariable(e2e.RKE2VersionVar)
+		Expect(rke2Version).ToNot(BeEmpty(), "RKE2 version is required")
+
+		credsSecretName := "cc-test99"
+		credsName := "az-ecm"
+		poolName := "az-test-pool"
+		clusterName := "az-cluster1"
+
+		lookupResult := &turtlesframework.RancherLookupUserResult{}
+		turtlesframework.RancherLookupUser(ctx, turtlesframework.RancherLookupUserInput{
+			Username:     "admin",
+			ClusterProxy: setupClusterResult.BootstrapClusterProxy,
+		}, lookupResult)
+
+		turtlesframework.CreateSecret(ctx, turtlesframework.CreateSecretInput{
+			Creator:   setupClusterResult.BootstrapClusterProxy.GetClient(),
+			Name:      credsSecretName,
+			Namespace: "cattle-global-data",
+			Type:      corev1.SecretTypeOpaque,
+			Data: map[string]string{
+				"azurecredentialConfig-clientId":       azClientId,
+				"azurecredentialConfig-clientSecret":   azClientSecret,
+				"azurecredentialConfig-environment":    "AzurePublicCloud",
+				"azurecredentialConfig-subscriptionId": azSubId,
+				"azurecredentialConfig-tenantId":       "",
+			},
+			Annotations: map[string]string{
+				"field.cattle.io/name":          credsName,
+				"provisioning.cattle.io/driver": "azure",
+				"field.cattle.io/creatorId":     lookupResult.User,
+			},
+			Labels: map[string]string{
+				"cattle.io/creator": "norman",
+			},
+		})
+
+		rkeConfig, err := envsubst.Eval(string(e2e.V2ProvAzureRkeConfig), func(s string) string {
+			switch s {
+			case "POOL_NAME":
+				return poolName
+			case "USER":
+				return lookupResult.User
+			default:
+				return os.Getenv(s)
+			}
+		})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, []byte(rkeConfig))).To(Succeed(), "Failed apply Digital Ocean RKE config")
+
+		cluster, err := envsubst.Eval(string(e2e.V2ProvAzureCluster), func(s string) string {
+			switch s {
+			case "CLUSTER_NAME":
+				return clusterName
+			case "USER":
+				return lookupResult.User
+			case "CREDENTIAL_SECRET":
+				return fmt.Sprintf("cattle-global-data:%s", credsSecretName)
+			case "KUBERNETES_VERSION":
+				return rke2Version
+			case "AZ_CONFIG_NAME":
+				return poolName
+			default:
+				return os.Getenv(s)
+			}
+		})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, []byte(cluster))).To(Succeed(), "Failed apply Digital Ocean cluster config")
+
+		By("Waiting for the rancher cluster record to appear")
+		rancherCluster := &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
+			Namespace: "fleet-default",
+			Name:      clusterName,
+		}}
+		Eventually(komega.Get(rancherCluster), e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
+
+		By("Waiting for the rancher cluster to have a deployed agent")
+		Eventually(komega.Object(rancherCluster), e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-v2prov-create")...).Should(HaveField("Status.AgentDeployed", BeTrue()))
+
+		By("Waiting for the rancher cluster to be ready")
+		Eventually(komega.Object(rancherCluster), e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.Ready", BeTrue()))
+
+		By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
+		rancherKubeconfig := &turtlesframework.RancherGetClusterKubeconfigResult{}
+		turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
+			Getter:           setupClusterResult.BootstrapClusterProxy.GetClient(),
+			SecretName:       fmt.Sprintf("%s-kubeconfig", rancherCluster.Name),
+			Namespace:        rancherCluster.Namespace,
+			RancherServerURL: hostName,
+			WriteToTempFile:  true,
+		}, rancherKubeconfig)
+
+		rancherConnectRes := &turtlesframework.RunCommandResult{}
+		turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{
+			Command: "kubectl",
+			Args: []string{
+				"--kubeconfig",
+				rancherKubeconfig.TempFilePath,
+				"get",
+				"nodes",
+				"--insecure-skip-tls-verify",
+			},
+		}, rancherConnectRes)
+		Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig")
+		Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code")
+
+		By("Deleting cluster from Rancher")
+		err = setupClusterResult.BootstrapClusterProxy.GetClient().Delete(ctx, rancherCluster)
+		Expect(err).NotTo(HaveOccurred(), "Failed to delete rancher cluster")
+
+		By("Waiting for the rancher cluster record to be removed")
+		Eventually(komega.Get(rancherCluster), e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-azure-delete")...).Should(MatchError(ContainSubstring("not found")), "Rancher cluster should be deleted")
+	})
+})

test/e2e/suites/embedded-capi-disabled/suite_test.go

@@ -0,0 +1,169 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2023 SUSE.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package embedded_capi_disabled
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	"github.com/rancher-sandbox/rancher-turtles/test/e2e"
+	turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework"
+	"github.com/rancher-sandbox/rancher-turtles/test/testenv"
+)
+
+// Test suite flags.
+var (
+	flagVals *e2e.FlagValues
+)
+
+// Test suite global vars.
+var (
+	// e2eConfig to be used for this test, read from configPath.
+	e2eConfig *clusterctl.E2EConfig
+
+	// clusterctlConfigPath to be used for this test, created by generating a clusterctl local repository
+	// with the providers specified in the configPath.
+	clusterctlConfigPath string
+
+	// hostName is the host name for the Rancher Manager server.
+	hostName string
+
+	ctx = context.Background()
+
+	setupClusterResult *testenv.SetupTestClusterResult
+)
+
+func init() {
+	flagVals = &e2e.FlagValues{}
+	e2e.InitFlags(flagVals)
+}
+
+func TestE2E(t *testing.T) {
+	RegisterFailHandler(Fail)
+
+	ctrl.SetLogger(klog.Background())
+
+	RunSpecs(t, "rancher-turtles-e2e-embedded-capi-disabled")
+}
+
+var _ = BeforeSuite(func() {
+	Expect(flagVals.ConfigPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.")
+	Expect(os.MkdirAll(flagVals.ArtifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", flagVals.ArtifactFolder)
+	Expect(flagVals.HelmBinaryPath).To(BeAnExistingFile(), "Invalid test suite argument. helm-binary-path should be an existing file.")
+	Expect(flagVals.ChartPath).To(BeAnExistingFile(), "Invalid test suite argument. chart-path should be an existing file.")
+
+	By(fmt.Sprintf("Loading the e2e test configuration from %q", flagVals.ConfigPath))
+	e2eConfig = e2e.LoadE2EConfig(flagVals.ConfigPath)
+
+	By(fmt.Sprintf("Creating a clusterctl config into %q", flagVals.ArtifactFolder))
+	clusterctlConfigPath = e2e.CreateClusterctlLocalRepository(ctx, e2eConfig, filepath.Join(flagVals.ArtifactFolder, "repository"))
+
+	hostName = e2eConfig.GetVariable(e2e.RancherHostnameVar)
+
+	setupClusterResult = testenv.SetupTestCluster(ctx, testenv.SetupTestClusterInput{
+		UseExistingCluster:   flagVals.UseExistingCluster,
+		E2EConfig:            e2eConfig,
+		ClusterctlConfigPath: clusterctlConfigPath,
+		Scheme:               e2e.InitScheme(),
+		ArtifactFolder:       flagVals.ArtifactFolder,
+		Hostname:             hostName,
+		KubernetesVersion:    e2eConfig.GetVariable(e2e.KubernetesVersionVar),
+		IsolatedMode:         flagVals.IsolatedMode,
+		HelmBinaryPath:       flagVals.HelmBinaryPath,
+	})
+
+	if flagVals.IsolatedMode {
+		hostName = setupClusterResult.IsolatedHostName
+	}
+	turtlesframework.Byf("Rancher hostname is %s", hostName)
+
+	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
+		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
+		HelmBinaryPath:           flagVals.HelmBinaryPath,
+		IsolatedMode:             flagVals.IsolatedMode,
+		NginxIngress:             e2e.NginxIngress,
+		NginxIngressNamespace:    e2e.NginxIngressNamespace,
+		IngressWaitInterval:      e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
+		NgrokApiKey:              e2eConfig.GetVariable(e2e.NgrokApiKeyVar),
+		NgrokAuthToken:           e2eConfig.GetVariable(e2e.NgrokAuthTokenVar),
+		NgrokPath:                e2eConfig.GetVariable(e2e.NgrokPathVar),
+		NgrokRepoName:            e2eConfig.GetVariable(e2e.NgrokRepoNameVar),
+		NgrokRepoURL:             e2eConfig.GetVariable(e2e.NgrokUrlVar),
+		DefaultIngressClassPatch: e2e.IngressClassPatch,
+	})
+
+	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
+		BootstrapClusterProxy:  setupClusterResult.BootstrapClusterProxy,
+		HelmBinaryPath:         flagVals.HelmBinaryPath,
+		InstallCertManager:     true,
+		CertManagerChartPath:   e2eConfig.GetVariable(e2e.CertManagerPathVar),
+		CertManagerUrl:         e2eConfig.GetVariable(e2e.CertManagerUrlVar),
+		CertManagerRepoName:    e2eConfig.GetVariable(e2e.CertManagerRepoNameVar),
+		RancherChartRepoName:   e2eConfig.GetVariable(e2e.RancherRepoNameVar),
+		RancherChartURL:        e2eConfig.GetVariable(e2e.RancherUrlVar),
+		RancherChartPath:       e2eConfig.GetVariable(e2e.RancherPathVar),
+		RancherVersion:         e2eConfig.GetVariable(e2e.RancherVersionVar),
+		RancherHost:            hostName,
+		RancherNamespace:       e2e.RancherNamespace,
+		RancherPassword:        e2eConfig.GetVariable(e2e.RancherPasswordVar),
+		RancherFeatures:        "embedded-cluster-api=false",
+		RancherPatches:         [][]byte{e2e.RancherSettingPatch},
+		RancherWaitInterval:    e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
+		ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
+		IsolatedMode:           flagVals.IsolatedMode,
+		RancherIngressConfig:   e2e.IngressConfig,
+		RancherServicePatch:    e2e.RancherServicePatch,
+		Variables:              e2eConfig.Variables,
+	})
+
+	testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
+		BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,
+		HelmBinaryPath:               flagVals.HelmBinaryPath,
+		ChartPath:                    flagVals.ChartPath,
+		CAPIProvidersSecretYAML:      e2e.CapiProvidersSecret,
+		CAPIProvidersYAML:            e2e.CapiProviders,
+		Namespace:                    turtlesframework.DefaultRancherTurtlesNamespace,
+		Image:                        "ghcr.io/rancher-sandbox/rancher-turtles-amd64",
+		Tag:                          "v0.0.1",
+		WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
+		AdditionalValues: map[string]string{
+			"cluster-api-operator.cert-manager.enabled":      "false",
+			"rancherTurtles.features.embedded-capi.disabled": "false",
+		},
+	})
+})
+
+var _ = AfterSuite(func() {
+	testenv.CleanupTestCluster(ctx, testenv.CleanupTestClusterInput{
+		SetupTestClusterResult: *setupClusterResult,
+		SkipCleanup:            flagVals.SkipCleanup,
+		ArtifactFolder:         flagVals.ArtifactFolder,
+	})
+})

test/testenv/rancher.go

@@ -167,7 +167,7 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
 		"replicas":                  "1",
 	}
 	if input.RancherFeatures != "" {
-		values["features"] = input.RancherFeatures
+		values["CATTLE_FEATURES"] = input.RancherFeatures
 	}
 	if input.RancherImageTag != "" {
 		values["rancherImageTag"] = input.RancherImageTag