From bd68ef8125dc551a61af0675972ba9b5217b137f Mon Sep 17 00:00:00 2001 From: "chang.qiangqiang" Date: Fri, 29 Nov 2024 17:53:57 +0800 Subject: [PATCH] fix(cluster): remove raleted metrics label when cluster is deleted Signed-off-by: chang.qiangqiang --- pkg/controllers/cluster/cluster_controller.go | 8 +++++++- pkg/metrics/cluster.go | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/cluster_controller.go b/pkg/controllers/cluster/cluster_controller.go index 0c0b4bdf6f2f..56b14dbf6821 100644 --- a/pkg/controllers/cluster/cluster_controller.go +++ b/pkg/controllers/cluster/cluster_controller.go @@ -45,6 +45,7 @@ import ( workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" "github.com/karmada-io/karmada/pkg/events" "github.com/karmada-io/karmada/pkg/features" + "github.com/karmada-io/karmada/pkg/metrics" "github.com/karmada-io/karmada/pkg/util" utilhelper "github.com/karmada-io/karmada/pkg/util/helper" "github.com/karmada-io/karmada/pkg/util/names" @@ -238,8 +239,13 @@ func (c *Controller) syncCluster(ctx context.Context, cluster *clusterv1alpha1.C return c.ensureFinalizer(ctx, cluster) } -func (c *Controller) removeCluster(ctx context.Context, cluster *clusterv1alpha1.Cluster) (controllerruntime.Result, error) { +func (c *Controller) removeCluster(ctx context.Context, cluster *clusterv1alpha1.Cluster) (ret controllerruntime.Result, retErr error) { // add terminating taint before cluster is deleted + defer func() { + if retErr == nil { + metrics.RemoveClusterStatus(cluster) + } + }() if err := c.updateClusterTaints(ctx, []*corev1.Taint{TerminatingTaintTemplate}, nil, cluster); err != nil { klog.ErrorS(err, "Failed to update terminating taint", "cluster", cluster.Name) return controllerruntime.Result{}, err diff --git a/pkg/metrics/cluster.go b/pkg/metrics/cluster.go index 3078d71476dd..7fe3766ee0b5 100644 --- a/pkg/metrics/cluster.go +++ b/pkg/metrics/cluster.go @@ -135,6 +135,20 @@ func RecordClusterSyncStatusDuration(cluster *v1alpha1.Cluster, startTime time.T clusterSyncStatusDuration.WithLabelValues(cluster.Name).Observe(utilmetrics.DurationInSeconds(startTime)) } +// RemoveClusterStatus removes the cluster status metrics when the cluster is deleted. +func RemoveClusterStatus(cluster *v1alpha1.Cluster) { + clusterReadyGauge.DeleteLabelValues(cluster.Name) + clusterTotalNodeNumberGauge.DeleteLabelValues(cluster.Name) + clusterReadyNodeNumberGauge.DeleteLabelValues(cluster.Name) + clusterMemoryAllocatableGauge.DeleteLabelValues(cluster.Name) + clusterCPUAllocatableGauge.DeleteLabelValues(cluster.Name) + clusterPodAllocatableGauge.DeleteLabelValues(cluster.Name) + clusterMemoryAllocatedGauge.DeleteLabelValues(cluster.Name) + clusterCPUAllocatedGauge.DeleteLabelValues(cluster.Name) + clusterPodAllocatedGauge.DeleteLabelValues(cluster.Name) + clusterSyncStatusDuration.DeleteLabelValues(cluster.Name) +} + // ClusterCollectors returns the collectors about clusters. func ClusterCollectors() []prometheus.Collector { return []prometheus.Collector{