From ddf4102d18072e1d035e3b83e751a54bfac605b9 Mon Sep 17 00:00:00 2001 From: Mohamed Awnallah Date: Thu, 31 Oct 2024 21:23:49 +0300 Subject: [PATCH] pkg/karmadactl/addons: unit test descheduler In this commit, we unit test descheduler addon on enabling, disabling, and status operations. In addition to this we refactor some code to utilize some utilities across karmadactl addons. Signed-off-by: Mohamed Awnallah --- .../addons/descheduler/descheduler.go | 2 +- .../addons/descheduler/descheduler_test.go | 265 ++++++++++++++++++ pkg/karmadactl/addons/estimator/estimator.go | 2 +- .../addons/metricsadapter/metricsadapter.go | 2 +- .../metricsadapter/metricsadapter_test.go | 33 +-- pkg/karmadactl/addons/search/search.go | 2 +- pkg/karmadactl/addons/utils/helpers.go | 67 +++++ pkg/karmadactl/register/register.go | 3 +- 8 files changed, 340 insertions(+), 36 deletions(-) create mode 100644 pkg/karmadactl/addons/descheduler/descheduler_test.go create mode 100644 pkg/karmadactl/addons/utils/helpers.go diff --git a/pkg/karmadactl/addons/descheduler/descheduler.go b/pkg/karmadactl/addons/descheduler/descheduler.go index 3d76f6058a58..b6c455212d40 100644 --- a/pkg/karmadactl/addons/descheduler/descheduler.go +++ b/pkg/karmadactl/addons/descheduler/descheduler.go @@ -76,7 +76,7 @@ var enableDescheduler = func(opts *addoninit.CommandAddonsEnableOption) error { return fmt.Errorf("create karmada descheduler deployment error: %v", err) } - if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaDeschedulerDeployment, opts.WaitComponentReadyTimeout); err != nil { + if err := addonutils.WaitForDeploymentRollout(opts.KubeClientSet, karmadaDeschedulerDeployment, opts.WaitComponentReadyTimeout); err != nil { return fmt.Errorf("wait karmada descheduler pod timeout: %v", err) } diff --git a/pkg/karmadactl/addons/descheduler/descheduler_test.go b/pkg/karmadactl/addons/descheduler/descheduler_test.go new file mode 100644 index 000000000000..d27d9181c8c2 --- /dev/null +++ b/pkg/karmadactl/addons/descheduler/descheduler_test.go @@ -0,0 +1,265 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package descheduler + +import ( + "context" + "fmt" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" + cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" +) + +func TestStatus(t *testing.T) { + name, namespace := addoninit.DeschedulerResourceName, "test" + var replicas int32 = 2 + tests := []struct { + name string + listOpts *addoninit.CommandAddonsListOption + prep func(*addoninit.CommandAddonsListOption) error + wantStatus string + wantErr bool + errMsg string + }{ + { + name: "Status_WithoutDescheduler_AddonDisabledStatus", + listOpts: &addoninit.CommandAddonsListOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + }, + prep: func(*addoninit.CommandAddonsListOption) error { return nil }, + wantStatus: addoninit.AddonDisabledStatus, + }, + { + name: "Status_WithNetworkIssue_AddonUnknownStatus", + listOpts: &addoninit.CommandAddonsListOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + }, + prep: func(listOpts *addoninit.CommandAddonsListOption) error { + return addonutils.SimulateNetworkErrorOnOp(listOpts.KubeClientSet, "get", "deployments") + }, + wantStatus: addoninit.AddonUnknownStatus, + wantErr: true, + errMsg: "unexpected error: encountered a network issue while get the deployments", + }, + { + name: "Status_ForKarmadaDeschedulerNotFullyAvailable_AddonUnhealthyStatus", + listOpts: &addoninit.CommandAddonsListOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + }, + prep: func(listOpts *addoninit.CommandAddonsListOption) error { + if err := createKarmadaDeschedulerDeployment(listOpts.KubeClientSet, replicas, listOpts.Namespace); err != nil { + return fmt.Errorf("failed to create karmada descheduler deployment, got error: %v", err) + } + return addonutils.SimulateDeploymentUnready(listOpts.KubeClientSet, name, listOpts.Namespace) + }, + wantStatus: addoninit.AddonUnhealthyStatus, + }, + { + name: "Status_WithAvailableKarmadaDeschedulerDeployment_AddonEnabledStatus", + listOpts: &addoninit.CommandAddonsListOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + }, + prep: func(listOpts *addoninit.CommandAddonsListOption) error { + if err := createKarmadaDeschedulerDeployment(listOpts.KubeClientSet, replicas, listOpts.Namespace); err != nil { + return fmt.Errorf("failed to create karmada descheduler deployment, got error: %v", err) + } + return nil + }, + wantStatus: addoninit.AddonEnabledStatus, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.prep(test.listOpts); err != nil { + t.Fatalf("failed to prep test env before checking on karmada descheduler addon status, got error: %v", err) + } + deschedulerAddonStatus, err := status(test.listOpts) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Fatalf("unexpected error, got: %v", err) + } + if err != nil && test.wantErr && !strings.Contains(err.Error(), test.errMsg) { + t.Errorf("expected error message %s to be in %s", test.errMsg, err.Error()) + } + if deschedulerAddonStatus != test.wantStatus { + t.Errorf("expected addon status to be %s, but got %s", test.wantStatus, deschedulerAddonStatus) + } + }) + } +} + +func TestEnableDescheduler(t *testing.T) { + name, namespace := addoninit.DeschedulerResourceName, "test" + var replicas int32 = 2 + tests := []struct { + name string + enableOpts *addoninit.CommandAddonsEnableOption + prep func() error + wantErr bool + errMsg string + }{ + { + name: "EnableDescheduler_WaitingForKarmadaDescheduler_Created", + enableOpts: &addoninit.CommandAddonsEnableOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + KarmadaDeschedulerReplicas: replicas, + }, + prep: func() error { + addonutils.WaitForDeploymentRollout = func(client clientset.Interface, _ *appsv1.Deployment, _ int) error { + _, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %s, got an error: %v", name, err) + } + return nil + } + return nil + }, + wantErr: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.prep(); err != nil { + t.Fatalf("failed to prep test environment before enabling descheduler, got an error: %v", err) + } + err := enableDescheduler(test.enableOpts) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Errorf("unexpected error, got: %v", err) + } + }) + } +} + +func TestDisableDescheduler(t *testing.T) { + name, namespace := addoninit.DeschedulerResourceName, "test" + client := fakeclientset.NewSimpleClientset() + var replicas int32 = 2 + tests := []struct { + name string + enableOpts *addoninit.CommandAddonsEnableOption + disableOpts *addoninit.CommandAddonsDisableOption + prep func(*addoninit.CommandAddonsEnableOption) error + verify func(clientset.Interface) error + wantErr bool + errMsg string + }{ + { + name: "DisableDescheduler_DisablingKarmadaDescheduler_Disabled", + enableOpts: &addoninit.CommandAddonsEnableOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: client, + }, + KarmadaDeschedulerReplicas: replicas, + }, + disableOpts: &addoninit.CommandAddonsDisableOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: client, + }, + }, + prep: func(enableOpts *addoninit.CommandAddonsEnableOption) error { + addonutils.WaitForDeploymentRollout = func(client clientset.Interface, _ *appsv1.Deployment, _ int) error { + _, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %s, got an error: %v", name, err) + } + return nil + } + if err := enableDescheduler(enableOpts); err != nil { + return fmt.Errorf("failed to enable descheduler, got an error: %v", err) + } + return nil + }, + verify: func(client clientset.Interface) error { + _, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil { + return fmt.Errorf("deployment %s was expected to be deleted, but it was still found", name) + } + return nil + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.prep(test.enableOpts); err != nil { + t.Fatalf("failed to prep test environment before disabling descheduler, got an error: %v", err) + } + err := disableDescheduler(test.disableOpts) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Errorf("unexpected error, got: %v", err) + } + if err := test.verify(client); err != nil { + t.Errorf("failed to verify disabling descheduler, got an error: %v", err) + } + }) + } +} + +// createKarmadaDeschedulerDeployment creates or updates a Deployment for the Karmada descheduler +// in the specified namespace with the provided number of replicas. +// It parses and decodes the template for the Deployment before applying it to the cluster. +func createKarmadaDeschedulerDeployment(c clientset.Interface, replicas int32, namespace string) error { + karmadaDeschedulerDeploymentBytes, err := addonutils.ParseTemplate(karmadaDeschedulerDeployment, DeploymentReplace{ + Namespace: namespace, + Replicas: ptr.To[int32](replicas), + }) + if err != nil { + return fmt.Errorf("error when parsing karmada descheduler deployment template: %v", err) + } + + karmadaDeschedulerDeployment := &appsv1.Deployment{} + if err = kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaDeschedulerDeploymentBytes, karmadaDeschedulerDeployment); err != nil { + return fmt.Errorf("failed to decode karmada descheduler deployment, got error: %v", err) + } + if err = cmdutil.CreateOrUpdateDeployment(c, karmadaDeschedulerDeployment); err != nil { + return fmt.Errorf("failed to create karmada descheduler deployment, got error: %v", err) + } + return nil +} diff --git a/pkg/karmadactl/addons/estimator/estimator.go b/pkg/karmadactl/addons/estimator/estimator.go index 00f88cdf8eac..449385452cc7 100644 --- a/pkg/karmadactl/addons/estimator/estimator.go +++ b/pkg/karmadactl/addons/estimator/estimator.go @@ -127,7 +127,7 @@ var enableEstimator = func(opts *addoninit.CommandAddonsEnableOption) error { return fmt.Errorf("create or update scheduler estimator deployment error: %v", err) } - if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaEstimatorDeployment, opts.WaitComponentReadyTimeout); err != nil { + if err := addonutils.WaitForDeploymentRollout(opts.KubeClientSet, karmadaEstimatorDeployment, opts.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } klog.Infof("Karmada scheduler estimator of member cluster %s is installed successfully.", opts.Cluster) diff --git a/pkg/karmadactl/addons/metricsadapter/metricsadapter.go b/pkg/karmadactl/addons/metricsadapter/metricsadapter.go index 49221e35f578..ed82e3d005a7 100644 --- a/pkg/karmadactl/addons/metricsadapter/metricsadapter.go +++ b/pkg/karmadactl/addons/metricsadapter/metricsadapter.go @@ -169,7 +169,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e return fmt.Errorf("create karmada metrics adapter deployment error: %v", err) } - if err = cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaMetricsAdapterDeployment, opts.WaitComponentReadyTimeout); err != nil { + if err = addonutils.WaitForDeploymentRollout(opts.KubeClientSet, karmadaMetricsAdapterDeployment, opts.WaitComponentReadyTimeout); err != nil { return fmt.Errorf("wait karmada metrics adapter pod status ready timeout: %v", err) } diff --git a/pkg/karmadactl/addons/metricsadapter/metricsadapter_test.go b/pkg/karmadactl/addons/metricsadapter/metricsadapter_test.go index ef0f89ea6d26..ed152b716534 100644 --- a/pkg/karmadactl/addons/metricsadapter/metricsadapter_test.go +++ b/pkg/karmadactl/addons/metricsadapter/metricsadapter_test.go @@ -24,12 +24,10 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" kuberuntime "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" fakeclientset "k8s.io/client-go/kubernetes/fake" clientsetscheme "k8s.io/client-go/kubernetes/scheme" - coretesting "k8s.io/client-go/testing" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" fakeAggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake" @@ -69,7 +67,7 @@ func TestStatus(t *testing.T) { }, }, prep: func(listOpts *addoninit.CommandAddonsListOption) error { - return simulateNetworkErrorOnOp(listOpts.KubeClientSet, "get", "deployments") + return addonutils.SimulateNetworkErrorOnOp(listOpts.KubeClientSet, "get", "deployments") }, wantStatus: addoninit.AddonUnknownStatus, wantErr: true, @@ -87,7 +85,7 @@ func TestStatus(t *testing.T) { if err := createKarmadaMetricsDeployment(listOpts.KubeClientSet, replicas, listOpts.Namespace); err != nil { return fmt.Errorf("failed to create karmada metrics deployment, got error: %v", err) } - return simulateKarmadaMetricsDeploymentUnready(listOpts.KubeClientSet, name, listOpts.Namespace) + return addonutils.SimulateDeploymentUnready(listOpts.KubeClientSet, name, listOpts.Namespace) }, wantStatus: addoninit.AddonUnhealthyStatus, }, @@ -167,15 +165,6 @@ func TestStatus(t *testing.T) { } } -// simulateNetworkErrorOnOp simulates a network error during the specified -// operation on a resource by prepending a reactor to the fake client. -func simulateNetworkErrorOnOp(c clientset.Interface, operation, resource string) error { - c.(*fakeclientset.Clientset).Fake.PrependReactor(operation, resource, func(coretesting.Action) (bool, runtime.Object, error) { - return true, nil, fmt.Errorf("unexpected error: encountered a network issue while %s the %s", operation, resource) - }) - return nil -} - // createKarmadaMetricsDeployment creates or updates a Deployment for the Karmada metrics adapter // in the specified namespace with the provided number of replicas. // It parses and decodes the template for the Deployment before applying it to the cluster. @@ -236,24 +225,6 @@ func updateAAAPIServicesCondition(services []*apiregistrationv1.APIService, a ag return nil } -// simulateKarmadaMetricsDeploymentUnready simulates a "not ready" status by incrementing the replicas -// of the specified Deployment, thus marking it as unready. This is useful for testing the handling -// of Deployment readiness in Karmada. -func simulateKarmadaMetricsDeploymentUnready(c clientset.Interface, name, namespace string) error { - deployment, err := c.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get deployment %s in namespace %s, got error: %v", name, namespace, err) - } - - deployment.Status.Replicas = *deployment.Spec.Replicas + 1 - _, err = c.AppsV1().Deployments(namespace).UpdateStatus(context.TODO(), deployment, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("failed to update replicas status of deployment %s in namespace %s, got error: %v", name, namespace, err) - } - - return nil -} - // createAndMarkAAAPIServicesAvailable creates the specified AA API services and then // updates their conditions to mark them as available, setting a "ConditionTrue" status. // This function is a combination of the creation and condition-setting operations for convenience. diff --git a/pkg/karmadactl/addons/search/search.go b/pkg/karmadactl/addons/search/search.go index 7b9fef39dde3..5b313a82e6f9 100644 --- a/pkg/karmadactl/addons/search/search.go +++ b/pkg/karmadactl/addons/search/search.go @@ -182,7 +182,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e return fmt.Errorf("create karmada search deployment error: %v", err) } - if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaSearchDeployment, opts.WaitComponentReadyTimeout); err != nil { + if err := addonutils.WaitForDeploymentRollout(opts.KubeClientSet, karmadaSearchDeployment, opts.WaitComponentReadyTimeout); err != nil { return fmt.Errorf("wait karmada search pod status ready timeout: %v", err) } diff --git a/pkg/karmadactl/addons/utils/helpers.go b/pkg/karmadactl/addons/utils/helpers.go new file mode 100644 index 000000000000..bb0a79b4546a --- /dev/null +++ b/pkg/karmadactl/addons/utils/helpers.go @@ -0,0 +1,67 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + + cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" +) + +var ( + // WaitForDeploymentRollout waits for the specified Deployment to reach its desired state within the given timeout. + // This blocks until the Deployment's observed generation and ready replicas match the desired state, + // ensuring it is fully rolled out. + WaitForDeploymentRollout = func(c clientset.Interface, dep *appsv1.Deployment, timeoutSeconds int) error { + return cmdutil.WaitForDeploymentRollout(c, dep, timeoutSeconds) + } +) + +// SimulateNetworkErrorOnOp simulates a network error during the specified +// operation on a resource by prepending a reactor to the fake client. +func SimulateNetworkErrorOnOp(c clientset.Interface, operation, resource string) error { + c.(*fakeclientset.Clientset).Fake.PrependReactor(operation, resource, func(coretesting.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("unexpected error: encountered a network issue while %s the %s", operation, resource) + }) + return nil +} + +// SimulateDeploymentUnready simulates a "not ready" status by incrementing the replicas +// of the specified Deployment, thus marking it as unready. This is useful for testing the handling +// of Deployment readiness in Karmada. +func SimulateDeploymentUnready(c clientset.Interface, name, namespace string) error { + deployment, err := c.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %s in namespace %s, got error: %v", name, namespace, err) + } + + deployment.Status.Replicas = *deployment.Spec.Replicas + 1 + _, err = c.AppsV1().Deployments(namespace).UpdateStatus(context.TODO(), deployment, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update replicas status of deployment %s in namespace %s, got error: %v", name, namespace, err) + } + + return nil +} diff --git a/pkg/karmadactl/register/register.go b/pkg/karmadactl/register/register.go index 852c2150d1b8..c07db7cc78f7 100644 --- a/pkg/karmadactl/register/register.go +++ b/pkg/karmadactl/register/register.go @@ -51,6 +51,7 @@ import ( "github.com/karmada-io/karmada/pkg/apis/cluster/validation" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" + addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" @@ -398,7 +399,7 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { return err } - if err := cmdutil.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { + if err := addonutils.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { return err }