Merge pull request #3838 from chaosi-zju/migration-e2e

test: add e2e test for seamless migration

test/e2e/seamless_migration_test.go

@@ -0,0 +1,234 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+
+	policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
+	workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
+	"github.com/karmada-io/karmada/pkg/util/names"
+	"github.com/karmada-io/karmada/test/e2e/framework"
+	"github.com/karmada-io/karmada/test/helper"
+)
+
+var _ = ginkgo.Describe("Seamless migration testing", func() {
+	var member1 string
+	var member1Client kubernetes.Interface
+
+	ginkgo.BeforeEach(func() {
+		member1 = framework.ClusterNames()[0]
+		member1Client = framework.GetClusterClient(member1)
+	})
+
+	// referring to related manual steps: https://github.com/karmada-io/karmada/pull/3821#issuecomment-1649238940
+	ginkgo.Context("Test migrate namespaced resource: Deployment", func() {
+		var deployment *appsv1.Deployment
+		var propagationPolicy *policyv1alpha1.PropagationPolicy
+		var bindingName string
+
+		ginkgo.BeforeEach(func() {
+			deployment = helper.NewDeployment(testNamespace, deploymentNamePrefix+rand.String(RandomStrLength))
+			propagationPolicy = helper.NewPropagationPolicy(deployment.Namespace, deployment.Name, []policyv1alpha1.ResourceSelector{
+				{
+					APIVersion: deployment.APIVersion,
+					Kind:       deployment.Kind,
+					Name:       deployment.Name,
+				},
+			}, policyv1alpha1.Placement{
+				ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}},
+			})
+			bindingName = names.GenerateBindingName(deployment.Kind, deployment.Name)
+		})
+
+		ginkgo.BeforeEach(func() {
+			// Create Deployment in member1 cluster
+			framework.CreateDeployment(member1Client, deployment)
+			// Create Deployment in karmada control plane
+			framework.CreateDeployment(kubeClient, deployment)
+			// Create PropagationPolicy in karmada control plane without conflictResolution field
+			framework.CreatePropagationPolicy(karmadaClient, propagationPolicy)
+
+			ginkgo.DeferCleanup(func() {
+				// Delete Deployment in karmada control plane
+				framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
+				// Delete PropagationPolicy in karmada control plane
+				framework.RemovePropagationPolicy(karmadaClient, propagationPolicy.Namespace, propagationPolicy.Name)
+
+				// Verify Deployment in member cluster will be deleted automatically after promotion since it has been deleted from Karmada
+				klog.Infof("Waiting for Deployment deleted from cluster(%s)", member1)
+				framework.WaitDeploymentDisappearOnCluster(member1, testNamespace, deployment.Name)
+			})
+		})
+
+		ginkgo.It("Verify migrate a Deployment from member cluster", func() {
+			// Step 1, Verify ResourceBinding got unHealthy for resource already exist
+			ginkgo.By(fmt.Sprintf("Verify ResourceBinding %s got unApplied for resource already exist", bindingName), func() {
+				klog.Infof("Waiting to verify ResourceBinding %s got unApplied for resource already exist", bindingName)
+				gomega.Eventually(func() bool {
+					binding, err := karmadaClient.WorkV1alpha2().ResourceBindings(deployment.Namespace).Get(context.TODO(), bindingName, metav1.GetOptions{})
+					if err != nil && apierrors.IsNotFound(err) {
+						return false
+					}
+					gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+
+					items := binding.Status.AggregatedStatus
+					return len(items) > 0 && items[0].Applied == false && items[0].Health != workv1alpha2.ResourceHealthy
+				}, pollTimeout, pollInterval).Should(gomega.Equal(true))
+			})
+
+			// Step 2, Update PropagationPolicy in karmada control plane with conflictResolution=Overwrite
+			ginkgo.By(fmt.Sprintf("Update PropagationPolicy %s in karmada control plane with conflictResolution=Overwrite", propagationPolicy.Name), func() {
+				propagationPolicy.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite
+				framework.UpdatePropagationPolicyWithSpec(karmadaClient, propagationPolicy.Namespace, propagationPolicy.Name, propagationPolicy.Spec)
+			})
+
+			// Step 3, Verify Deployment Replicas all ready and ResourceBinding got Healthy for overwriting conflict resource
+			ginkgo.By(fmt.Sprintf("Verify Deployment Replicas all ready and ResourceBinding %s got Healthy for overwriting conflict resource", bindingName), func() {
+				klog.Infof("Waiting for Deployment ready on Karmada control plane")
+				framework.WaitDeploymentStatus(kubeClient, deployment, *deployment.Spec.Replicas)
+
+				binding, err := karmadaClient.WorkV1alpha2().ResourceBindings(deployment.Namespace).Get(context.TODO(), bindingName, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+
+				items := binding.Status.AggregatedStatus
+				gomega.Expect(len(items)).ShouldNot(gomega.Equal(0))
+				gomega.Expect(items[0].Applied).Should(gomega.BeTrue())
+				gomega.Expect(items[0].Health).Should(gomega.Equal(workv1alpha2.ResourceHealthy))
+			})
+		})
+	})
+
+	ginkgo.Context("Test migrate cluster resources: ClusterRole", func() {
+		var clusterRoleName string
+		var clusterRole *rbacv1.ClusterRole
+		var cpp *policyv1alpha1.ClusterPropagationPolicy
+		var bindingName string
+
+		ginkgo.BeforeEach(func() {
+			clusterRoleName = clusterRoleNamePrefix + rand.String(RandomStrLength)
+			clusterRole = helper.NewClusterRole(clusterRoleName, []rbacv1.PolicyRule{
+				{
+					APIGroups:     []string{"cluster.karmada.io"},
+					Verbs:         []string{"*"},
+					Resources:     []string{"clusters/proxy"},
+					ResourceNames: []string{member1},
+				},
+			})
+			cpp = helper.NewClusterPropagationPolicy(clusterRole.Name, []policyv1alpha1.ResourceSelector{
+				{
+					APIVersion: clusterRole.APIVersion,
+					Kind:       clusterRole.Kind,
+					Name:       clusterRole.Name,
+				},
+			}, policyv1alpha1.Placement{
+				ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}},
+			})
+			cpp.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite
+			bindingName = names.GenerateBindingName(clusterRole.Kind, clusterRole.Name)
+		})
+
+		ginkgo.BeforeEach(func() {
+			// Create Deployment in member1 cluster
+			framework.CreateClusterRole(member1Client, clusterRole)
+			// Create Deployment in karmada control plane
+			framework.CreateClusterRole(kubeClient, clusterRole)
+			// Create PropagationPolicy in karmada control plane without conflictResolution field
+			framework.CreateClusterPropagationPolicy(karmadaClient, cpp)
+
+			ginkgo.DeferCleanup(func() {
+				// Delete ClusterRole in karmada control plane
+				framework.RemoveClusterRole(kubeClient, clusterRoleName)
+				// Delete ClusterPropagationPolicy in karmada control plane
+				framework.RemoveClusterPropagationPolicy(karmadaClient, cpp.Name)
+
+				// Verify ClusterRole in member cluster will be deleted automatically after promotion since it has been deleted from Karmada
+				klog.Infof("Waiting for ClusterRole deleted from cluster(%s)", member1)
+				framework.WaitClusterRoleDisappearOnCluster(member1, clusterRoleName)
+			})
+		})
+
+		ginkgo.It("Verify migrate a ClusterRole from member cluster", func() {
+			ginkgo.By(fmt.Sprintf("Verify ClusterResourceBinding %s got Applied by overwriting conflict resource", bindingName), func() {
+				klog.Infof("Waiting to verify ResourceBinding %s got Applied by overwriting conflict resource", bindingName)
+				gomega.Eventually(func() bool {
+					framework.WaitClusterRolePresentOnClusterFitWith(member1, clusterRoleName, func(clusterRole *rbacv1.ClusterRole) bool {
+						return true
+					})
+					_, e1 := kubeClient.RbacV1().ClusterRoles().Get(context.TODO(), clusterRoleName, metav1.GetOptions{})
+					binding, e2 := karmadaClient.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), bindingName, metav1.GetOptions{})
+
+					return e1 == nil && e2 == nil && len(binding.Status.AggregatedStatus) > 0 && binding.Status.AggregatedStatus[0].Applied
+				}, pollTimeout, pollInterval).Should(gomega.Equal(true))
+			})
+		})
+	})
+
+	ginkgo.Context("Test migrate namespaced resource: Service (NodePort)", func() {
+		var serviceName string
+		var service *corev1.Service
+		var pp *policyv1alpha1.PropagationPolicy
+		var bindingName string
+
+		ginkgo.BeforeEach(func() {
+			serviceName = serviceNamePrefix + rand.String(RandomStrLength)
+			service = helper.NewService(testNamespace, serviceName, corev1.ServiceTypeNodePort)
+			pp = helper.NewPropagationPolicy(testNamespace, service.Name, []policyv1alpha1.ResourceSelector{
+				{
+					APIVersion: service.APIVersion,
+					Kind:       service.Kind,
+					Name:       service.Name,
+				},
+			}, policyv1alpha1.Placement{
+				ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}},
+			})
+			pp.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite
+			bindingName = names.GenerateBindingName(service.Kind, service.Name)
+		})
+
+		ginkgo.BeforeEach(func() {
+			// Create Deployment in member1 cluster
+			framework.CreateService(member1Client, service)
+			// Create Deployment in karmada control plane
+			framework.CreateService(kubeClient, service)
+			// Create PropagationPolicy in karmada control plane without conflictResolution field
+			framework.CreatePropagationPolicy(karmadaClient, pp)
+
+			ginkgo.DeferCleanup(func() {
+				// Delete Service in karmada control plane
+				framework.RemoveService(kubeClient, testNamespace, serviceName)
+				// Delete PropagationPolicy in karmada control plane
+				framework.RemovePropagationPolicy(karmadaClient, testNamespace, pp.Name)
+
+				// Verify Service in member cluster will be deleted automatically after promotion since it has been deleted from Karmada
+				klog.Infof("Waiting for Service deleted from cluster(%s)", member1)
+				framework.WaitServiceDisappearOnCluster(member1, testNamespace, serviceName)
+			})
+		})
+
+		ginkgo.It("Verify migrate a Service from member cluster", func() {
+			ginkgo.By(fmt.Sprintf("Verify PropagationPolicy %s got Applied by overwriting conflict resource", bindingName), func() {
+				klog.Infof("Waiting to verify ResourceBinding %s got Applied by overwriting conflict resource", bindingName)
+				gomega.Eventually(func() bool {
+					framework.WaitServicePresentOnClusterFitWith(member1, testNamespace, serviceName, func(service *corev1.Service) bool {
+						return true
+					})
+					_, e1 := kubeClient.CoreV1().Services(testNamespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
+					binding, e2 := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Get(context.TODO(), bindingName, metav1.GetOptions{})
+
+					return e1 == nil && e2 == nil && len(binding.Status.AggregatedStatus) > 0 && binding.Status.AggregatedStatus[0].Applied
+				}, pollTimeout, pollInterval).Should(gomega.Equal(true))
+			})
+		})
+	})
+})