fix(cluster): remove raleted metrics label when cluster is deleted

Signed-off-by: chang.qiangqiang <chang.qiangqiang@immomo.com>
karmada-io · Nov 29, 2024 · bd68ef8 · bd68ef8
1 parent 88c3e44
commit bd68ef8
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 1 deletion.
diff --git a/pkg/controllers/cluster/cluster_controller.go b/pkg/controllers/cluster/cluster_controller.go
@@ -45,6 +45,7 @@ import (
 	workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
 	"github.com/karmada-io/karmada/pkg/events"
 	"github.com/karmada-io/karmada/pkg/features"
+	"github.com/karmada-io/karmada/pkg/metrics"
 	"github.com/karmada-io/karmada/pkg/util"
 	utilhelper "github.com/karmada-io/karmada/pkg/util/helper"
 	"github.com/karmada-io/karmada/pkg/util/names"
@@ -238,8 +239,13 @@ func (c *Controller) syncCluster(ctx context.Context, cluster *clusterv1alpha1.C
 	return c.ensureFinalizer(ctx, cluster)
 }
 
-func (c *Controller) removeCluster(ctx context.Context, cluster *clusterv1alpha1.Cluster) (controllerruntime.Result, error) {
+func (c *Controller) removeCluster(ctx context.Context, cluster *clusterv1alpha1.Cluster) (ret controllerruntime.Result, retErr error) {
 	// add terminating taint before cluster is deleted
+	defer func() {
+		if retErr == nil {
+			metrics.RemoveClusterStatus(cluster)
+		}
+	}()
 	if err := c.updateClusterTaints(ctx, []*corev1.Taint{TerminatingTaintTemplate}, nil, cluster); err != nil {
 		klog.ErrorS(err, "Failed to update terminating taint", "cluster", cluster.Name)
 		return controllerruntime.Result{}, err

diff --git a/pkg/metrics/cluster.go b/pkg/metrics/cluster.go
@@ -135,6 +135,20 @@ func RecordClusterSyncStatusDuration(cluster *v1alpha1.Cluster, startTime time.T
 	clusterSyncStatusDuration.WithLabelValues(cluster.Name).Observe(utilmetrics.DurationInSeconds(startTime))
 }
 
+// RemoveClusterStatus removes the cluster status metrics when the cluster is deleted.
+func RemoveClusterStatus(cluster *v1alpha1.Cluster) {
+	clusterReadyGauge.DeleteLabelValues(cluster.Name)
+	clusterTotalNodeNumberGauge.DeleteLabelValues(cluster.Name)
+	clusterReadyNodeNumberGauge.DeleteLabelValues(cluster.Name)
+	clusterMemoryAllocatableGauge.DeleteLabelValues(cluster.Name)
+	clusterCPUAllocatableGauge.DeleteLabelValues(cluster.Name)
+	clusterPodAllocatableGauge.DeleteLabelValues(cluster.Name)
+	clusterMemoryAllocatedGauge.DeleteLabelValues(cluster.Name)
+	clusterCPUAllocatedGauge.DeleteLabelValues(cluster.Name)
+	clusterPodAllocatedGauge.DeleteLabelValues(cluster.Name)
+	clusterSyncStatusDuration.DeleteLabelValues(cluster.Name)
+}
+
 // ClusterCollectors returns the collectors about clusters.
 func ClusterCollectors() []prometheus.Collector {
 	return []prometheus.Collector{