diff --git a/pkg/karmadactl/addons/descheduler/descheduler.go b/pkg/karmadactl/addons/descheduler/descheduler.go index 3d76f6058a58..b6c455212d40 100644 --- a/pkg/karmadactl/addons/descheduler/descheduler.go +++ b/pkg/karmadactl/addons/descheduler/descheduler.go @@ -76,7 +76,7 @@ var enableDescheduler = func(opts *addoninit.CommandAddonsEnableOption) error { return fmt.Errorf("create karmada descheduler deployment error: %v", err) } - if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaDeschedulerDeployment, opts.WaitComponentReadyTimeout); err != nil { + if err := addonutils.WaitForDeploymentRollout(opts.KubeClientSet, karmadaDeschedulerDeployment, opts.WaitComponentReadyTimeout); err != nil { return fmt.Errorf("wait karmada descheduler pod timeout: %v", err) } diff --git a/pkg/karmadactl/addons/descheduler/descheduler_test.go b/pkg/karmadactl/addons/descheduler/descheduler_test.go new file mode 100644 index 000000000000..d27d9181c8c2 --- /dev/null +++ b/pkg/karmadactl/addons/descheduler/descheduler_test.go @@ -0,0 +1,265 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package descheduler + +import ( + "context" + "fmt" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" + cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" +) + +func TestStatus(t *testing.T) { + name, namespace := addoninit.DeschedulerResourceName, "test" + var replicas int32 = 2 + tests := []struct { + name string + listOpts *addoninit.CommandAddonsListOption + prep func(*addoninit.CommandAddonsListOption) error + wantStatus string + wantErr bool + errMsg string + }{ + { + name: "Status_WithoutDescheduler_AddonDisabledStatus", + listOpts: &addoninit.CommandAddonsListOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + }, + prep: func(*addoninit.CommandAddonsListOption) error { return nil }, + wantStatus: addoninit.AddonDisabledStatus, + }, + { + name: "Status_WithNetworkIssue_AddonUnknownStatus", + listOpts: &addoninit.CommandAddonsListOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + }, + prep: func(listOpts *addoninit.CommandAddonsListOption) error { + return addonutils.SimulateNetworkErrorOnOp(listOpts.KubeClientSet, "get", "deployments") + }, + wantStatus: addoninit.AddonUnknownStatus, + wantErr: true, + errMsg: "unexpected error: encountered a network issue while get the deployments", + }, + { + name: "Status_ForKarmadaDeschedulerNotFullyAvailable_AddonUnhealthyStatus", + listOpts: &addoninit.CommandAddonsListOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + }, + prep: func(listOpts *addoninit.CommandAddonsListOption) error { + if err := createKarmadaDeschedulerDeployment(listOpts.KubeClientSet, replicas, listOpts.Namespace); err != nil { + return fmt.Errorf("failed to create karmada descheduler deployment, got error: %v", err) + } + return addonutils.SimulateDeploymentUnready(listOpts.KubeClientSet, name, listOpts.Namespace) + }, + wantStatus: addoninit.AddonUnhealthyStatus, + }, + { + name: "Status_WithAvailableKarmadaDeschedulerDeployment_AddonEnabledStatus", + listOpts: &addoninit.CommandAddonsListOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + }, + prep: func(listOpts *addoninit.CommandAddonsListOption) error { + if err := createKarmadaDeschedulerDeployment(listOpts.KubeClientSet, replicas, listOpts.Namespace); err != nil { + return fmt.Errorf("failed to create karmada descheduler deployment, got error: %v", err) + } + return nil + }, + wantStatus: addoninit.AddonEnabledStatus, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.prep(test.listOpts); err != nil { + t.Fatalf("failed to prep test env before checking on karmada descheduler addon status, got error: %v", err) + } + deschedulerAddonStatus, err := status(test.listOpts) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Fatalf("unexpected error, got: %v", err) + } + if err != nil && test.wantErr && !strings.Contains(err.Error(), test.errMsg) { + t.Errorf("expected error message %s to be in %s", test.errMsg, err.Error()) + } + if deschedulerAddonStatus != test.wantStatus { + t.Errorf("expected addon status to be %s, but got %s", test.wantStatus, deschedulerAddonStatus) + } + }) + } +} + +func TestEnableDescheduler(t *testing.T) { + name, namespace := addoninit.DeschedulerResourceName, "test" + var replicas int32 = 2 + tests := []struct { + name string + enableOpts *addoninit.CommandAddonsEnableOption + prep func() error + wantErr bool + errMsg string + }{ + { + name: "EnableDescheduler_WaitingForKarmadaDescheduler_Created", + enableOpts: &addoninit.CommandAddonsEnableOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: fakeclientset.NewSimpleClientset(), + }, + KarmadaDeschedulerReplicas: replicas, + }, + prep: func() error { + addonutils.WaitForDeploymentRollout = func(client clientset.Interface, _ *appsv1.Deployment, _ int) error { + _, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %s, got an error: %v", name, err) + } + return nil + } + return nil + }, + wantErr: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.prep(); err != nil { + t.Fatalf("failed to prep test environment before enabling descheduler, got an error: %v", err) + } + err := enableDescheduler(test.enableOpts) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Errorf("unexpected error, got: %v", err) + } + }) + } +} + +func TestDisableDescheduler(t *testing.T) { + name, namespace := addoninit.DeschedulerResourceName, "test" + client := fakeclientset.NewSimpleClientset() + var replicas int32 = 2 + tests := []struct { + name string + enableOpts *addoninit.CommandAddonsEnableOption + disableOpts *addoninit.CommandAddonsDisableOption + prep func(*addoninit.CommandAddonsEnableOption) error + verify func(clientset.Interface) error + wantErr bool + errMsg string + }{ + { + name: "DisableDescheduler_DisablingKarmadaDescheduler_Disabled", + enableOpts: &addoninit.CommandAddonsEnableOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: client, + }, + KarmadaDeschedulerReplicas: replicas, + }, + disableOpts: &addoninit.CommandAddonsDisableOption{ + GlobalCommandOptions: addoninit.GlobalCommandOptions{ + Namespace: namespace, + KubeClientSet: client, + }, + }, + prep: func(enableOpts *addoninit.CommandAddonsEnableOption) error { + addonutils.WaitForDeploymentRollout = func(client clientset.Interface, _ *appsv1.Deployment, _ int) error { + _, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %s, got an error: %v", name, err) + } + return nil + } + if err := enableDescheduler(enableOpts); err != nil { + return fmt.Errorf("failed to enable descheduler, got an error: %v", err) + } + return nil + }, + verify: func(client clientset.Interface) error { + _, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil { + return fmt.Errorf("deployment %s was expected to be deleted, but it was still found", name) + } + return nil + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.prep(test.enableOpts); err != nil { + t.Fatalf("failed to prep test environment before disabling descheduler, got an error: %v", err) + } + err := disableDescheduler(test.disableOpts) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Errorf("unexpected error, got: %v", err) + } + if err := test.verify(client); err != nil { + t.Errorf("failed to verify disabling descheduler, got an error: %v", err) + } + }) + } +} + +// createKarmadaDeschedulerDeployment creates or updates a Deployment for the Karmada descheduler +// in the specified namespace with the provided number of replicas. +// It parses and decodes the template for the Deployment before applying it to the cluster. +func createKarmadaDeschedulerDeployment(c clientset.Interface, replicas int32, namespace string) error { + karmadaDeschedulerDeploymentBytes, err := addonutils.ParseTemplate(karmadaDeschedulerDeployment, DeploymentReplace{ + Namespace: namespace, + Replicas: ptr.To[int32](replicas), + }) + if err != nil { + return fmt.Errorf("error when parsing karmada descheduler deployment template: %v", err) + } + + karmadaDeschedulerDeployment := &appsv1.Deployment{} + if err = kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaDeschedulerDeploymentBytes, karmadaDeschedulerDeployment); err != nil { + return fmt.Errorf("failed to decode karmada descheduler deployment, got error: %v", err) + } + if err = cmdutil.CreateOrUpdateDeployment(c, karmadaDeschedulerDeployment); err != nil { + return fmt.Errorf("failed to create karmada descheduler deployment, got error: %v", err) + } + return nil +} diff --git a/pkg/karmadactl/addons/estimator/estimator.go b/pkg/karmadactl/addons/estimator/estimator.go index 00f88cdf8eac..449385452cc7 100644 --- a/pkg/karmadactl/addons/estimator/estimator.go +++ b/pkg/karmadactl/addons/estimator/estimator.go @@ -127,7 +127,7 @@ var enableEstimator = func(opts *addoninit.CommandAddonsEnableOption) error { return fmt.Errorf("create or update scheduler estimator deployment error: %v", err) } - if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaEstimatorDeployment, opts.WaitComponentReadyTimeout); err != nil { + if err := addonutils.WaitForDeploymentRollout(opts.KubeClientSet, karmadaEstimatorDeployment, opts.WaitComponentReadyTimeout); err != nil { klog.Warning(err) } klog.Infof("Karmada scheduler estimator of member cluster %s is installed successfully.", opts.Cluster) diff --git a/pkg/karmadactl/addons/metricsadapter/metricsadapter.go b/pkg/karmadactl/addons/metricsadapter/metricsadapter.go index 49221e35f578..ed82e3d005a7 100644 --- a/pkg/karmadactl/addons/metricsadapter/metricsadapter.go +++ b/pkg/karmadactl/addons/metricsadapter/metricsadapter.go @@ -169,7 +169,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e return fmt.Errorf("create karmada metrics adapter deployment error: %v", err) } - if err = cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaMetricsAdapterDeployment, opts.WaitComponentReadyTimeout); err != nil { + if err = addonutils.WaitForDeploymentRollout(opts.KubeClientSet, karmadaMetricsAdapterDeployment, opts.WaitComponentReadyTimeout); err != nil { return fmt.Errorf("wait karmada metrics adapter pod status ready timeout: %v", err) } diff --git a/pkg/karmadactl/addons/metricsadapter/metricsadapter_test.go b/pkg/karmadactl/addons/metricsadapter/metricsadapter_test.go index ef0f89ea6d26..ed152b716534 100644 --- a/pkg/karmadactl/addons/metricsadapter/metricsadapter_test.go +++ b/pkg/karmadactl/addons/metricsadapter/metricsadapter_test.go @@ -24,12 +24,10 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" kuberuntime "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" fakeclientset "k8s.io/client-go/kubernetes/fake" clientsetscheme "k8s.io/client-go/kubernetes/scheme" - coretesting "k8s.io/client-go/testing" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" fakeAggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake" @@ -69,7 +67,7 @@ func TestStatus(t *testing.T) { }, }, prep: func(listOpts *addoninit.CommandAddonsListOption) error { - return simulateNetworkErrorOnOp(listOpts.KubeClientSet, "get", "deployments") + return addonutils.SimulateNetworkErrorOnOp(listOpts.KubeClientSet, "get", "deployments") }, wantStatus: addoninit.AddonUnknownStatus, wantErr: true, @@ -87,7 +85,7 @@ func TestStatus(t *testing.T) { if err := createKarmadaMetricsDeployment(listOpts.KubeClientSet, replicas, listOpts.Namespace); err != nil { return fmt.Errorf("failed to create karmada metrics deployment, got error: %v", err) } - return simulateKarmadaMetricsDeploymentUnready(listOpts.KubeClientSet, name, listOpts.Namespace) + return addonutils.SimulateDeploymentUnready(listOpts.KubeClientSet, name, listOpts.Namespace) }, wantStatus: addoninit.AddonUnhealthyStatus, }, @@ -167,15 +165,6 @@ func TestStatus(t *testing.T) { } } -// simulateNetworkErrorOnOp simulates a network error during the specified -// operation on a resource by prepending a reactor to the fake client. -func simulateNetworkErrorOnOp(c clientset.Interface, operation, resource string) error { - c.(*fakeclientset.Clientset).Fake.PrependReactor(operation, resource, func(coretesting.Action) (bool, runtime.Object, error) { - return true, nil, fmt.Errorf("unexpected error: encountered a network issue while %s the %s", operation, resource) - }) - return nil -} - // createKarmadaMetricsDeployment creates or updates a Deployment for the Karmada metrics adapter // in the specified namespace with the provided number of replicas. // It parses and decodes the template for the Deployment before applying it to the cluster. @@ -236,24 +225,6 @@ func updateAAAPIServicesCondition(services []*apiregistrationv1.APIService, a ag return nil } -// simulateKarmadaMetricsDeploymentUnready simulates a "not ready" status by incrementing the replicas -// of the specified Deployment, thus marking it as unready. This is useful for testing the handling -// of Deployment readiness in Karmada. -func simulateKarmadaMetricsDeploymentUnready(c clientset.Interface, name, namespace string) error { - deployment, err := c.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get deployment %s in namespace %s, got error: %v", name, namespace, err) - } - - deployment.Status.Replicas = *deployment.Spec.Replicas + 1 - _, err = c.AppsV1().Deployments(namespace).UpdateStatus(context.TODO(), deployment, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("failed to update replicas status of deployment %s in namespace %s, got error: %v", name, namespace, err) - } - - return nil -} - // createAndMarkAAAPIServicesAvailable creates the specified AA API services and then // updates their conditions to mark them as available, setting a "ConditionTrue" status. // This function is a combination of the creation and condition-setting operations for convenience. diff --git a/pkg/karmadactl/addons/search/search.go b/pkg/karmadactl/addons/search/search.go index 7b9fef39dde3..5b313a82e6f9 100644 --- a/pkg/karmadactl/addons/search/search.go +++ b/pkg/karmadactl/addons/search/search.go @@ -182,7 +182,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e return fmt.Errorf("create karmada search deployment error: %v", err) } - if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaSearchDeployment, opts.WaitComponentReadyTimeout); err != nil { + if err := addonutils.WaitForDeploymentRollout(opts.KubeClientSet, karmadaSearchDeployment, opts.WaitComponentReadyTimeout); err != nil { return fmt.Errorf("wait karmada search pod status ready timeout: %v", err) } diff --git a/pkg/karmadactl/addons/utils/helpers.go b/pkg/karmadactl/addons/utils/helpers.go new file mode 100644 index 000000000000..bb0a79b4546a --- /dev/null +++ b/pkg/karmadactl/addons/utils/helpers.go @@ -0,0 +1,67 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + + cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" +) + +var ( + // WaitForDeploymentRollout waits for the specified Deployment to reach its desired state within the given timeout. + // This blocks until the Deployment's observed generation and ready replicas match the desired state, + // ensuring it is fully rolled out. + WaitForDeploymentRollout = func(c clientset.Interface, dep *appsv1.Deployment, timeoutSeconds int) error { + return cmdutil.WaitForDeploymentRollout(c, dep, timeoutSeconds) + } +) + +// SimulateNetworkErrorOnOp simulates a network error during the specified +// operation on a resource by prepending a reactor to the fake client. +func SimulateNetworkErrorOnOp(c clientset.Interface, operation, resource string) error { + c.(*fakeclientset.Clientset).Fake.PrependReactor(operation, resource, func(coretesting.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("unexpected error: encountered a network issue while %s the %s", operation, resource) + }) + return nil +} + +// SimulateDeploymentUnready simulates a "not ready" status by incrementing the replicas +// of the specified Deployment, thus marking it as unready. This is useful for testing the handling +// of Deployment readiness in Karmada. +func SimulateDeploymentUnready(c clientset.Interface, name, namespace string) error { + deployment, err := c.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get deployment %s in namespace %s, got error: %v", name, namespace, err) + } + + deployment.Status.Replicas = *deployment.Spec.Replicas + 1 + _, err = c.AppsV1().Deployments(namespace).UpdateStatus(context.TODO(), deployment, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update replicas status of deployment %s in namespace %s, got error: %v", name, namespace, err) + } + + return nil +} diff --git a/pkg/karmadactl/register/register.go b/pkg/karmadactl/register/register.go index 852c2150d1b8..c07db7cc78f7 100644 --- a/pkg/karmadactl/register/register.go +++ b/pkg/karmadactl/register/register.go @@ -51,6 +51,7 @@ import ( "github.com/karmada-io/karmada/pkg/apis/cluster/validation" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" + addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" @@ -398,7 +399,7 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { return err } - if err := cmdutil.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { + if err := addonutils.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { return err }