Effectiveness metrics (#410)

* add met metrics * add counter metrics * fix error
mercari · Sep 18, 2024 · b9f4a1c · b9f4a1c
1 parent eb8e267
commit b9f4a1c
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 0 deletions.
diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go
@@ -446,6 +446,16 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet
 	hpa.Spec.MinReplicas = &minToActuallyApply
 	if tortoise.Spec.UpdateMode != autoscalingv1beta3.UpdateModeOff && recordMetrics {
 		// We don't want to record applied* metric when UpdateMode is Off.
+		netChangeMaxReplicas := float64(hpa.Spec.MaxReplicas - recommendMax)
+		netChangeMinReplicas := float64(*hpa.Spec.MinReplicas) - float64(recommendMin)
+		if netChangeMaxReplicas > 0 || netChangeMinReplicas < 0 {
+			metrics.IncreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1)
+		}
+		if netChangeMaxReplicas < 0 || netChangeMinReplicas > 0 {
+			metrics.DecreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1)
+		}
+		metrics.NetHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMinReplicas)
+		metrics.NetHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMaxReplicas)
 		metrics.AppliedHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(*hpa.Spec.MinReplicas))
 		metrics.AppliedHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(hpa.Spec.MaxReplicas))
 	}

diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -46,6 +46,36 @@ var (
 		Help: "memory request (byte) that tortoises actually applys",
 	}, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"})
 
+	DecreaseApplyCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "decrease_apply_counter",
+		Help: "counter for number of resource decreases applied by tortoise",
+	}, []string{"tortoise_name", "namespace"})
+
+	IncreaseApplyCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "increase_apply_counter",
+		Help: "counter for number of resource increases applied by tortoise",
+	}, []string{"tortoise_name", "namespace"})
+
+	NetHPAMinReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "net_hpa_minreplicas",
+		Help: "net hpa minReplicas that tortoises actually applys to hpa",
+	}, []string{"tortoise_name", "namespace", "hpa_name"})
+
+	NetHPAMaxReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "net_hpa_maxreplicas",
+		Help: "net hpa maxReplicas that tortoises actually applys to hpa",
+	}, []string{"tortoise_name", "namespace", "hpa_name"})
+
+	NetCPURequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "net_cpu_request",
+		Help: "net cpu request (millicore) that tortoises actually applys",
+	}, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"})
+
+	NetMemoryRequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "net_memory_request",
+		Help: "net memory request (byte) that tortoises actually applys",
+	}, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"})
+
 	ProposedHPATargetUtilization = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "proposed_hpa_utilization_target",
 		Help: "recommended hpa utilization target values that tortoises propose",
@@ -87,6 +117,12 @@ func init() {
 		AppliedHPAMinReplicas,
 		AppliedCPURequest,
 		AppliedMemoryRequest,
+		IncreaseApplyCounter,
+		DecreaseApplyCounter,
+		NetHPAMaxReplicas,
+		NetHPAMinReplicas,
+		NetCPURequest,
+		NetMemoryRequest,
 		ProposedHPATargetUtilization,
 		ProposedHPAMinReplicas,
 		ProposedHPAMaxReplicas,

diff --git a/pkg/tortoise/tortoise.go b/pkg/tortoise/tortoise.go
@@ -665,6 +665,14 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T
 
 	oldTortoise := tortoise.DeepCopy()
 
+	oldRequestMap := map[string]map[corev1.ResourceName]resource.Quantity{}
+	for _, r := range tortoise.Status.Conditions.ContainerResourceRequests {
+		oldRequestMap[r.ContainerName] = map[corev1.ResourceName]resource.Quantity{}
+		for resourcename, value := range r.Resource {
+			oldRequestMap[r.ContainerName][resourcename] = value
+		}
+	}
+
 	newRequests := make([]v1beta3.ContainerResourceRequests, 0, len(tortoise.Status.Recommendations.Vertical.ContainerResourceRecommendation))
 	for _, r := range tortoise.Status.Recommendations.Vertical.ContainerResourceRecommendation {
 		recommendation := r.RecommendedResource.DeepCopy()
@@ -759,12 +767,22 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T
 	for _, r := range tortoise.Status.Conditions.ContainerResourceRequests {
 		// only record metrics once in every reconcile loop.
 		for resourcename, value := range r.Resource {
+			oldRequest := oldRequestMap[r.ContainerName][resourcename]
+			netChange := float64(oldRequest.MilliValue() - value.MilliValue())
 			if resourcename == corev1.ResourceCPU {
 				// We don't want to record applied* metric when UpdateMode is Off.
 				metrics.AppliedCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.MilliValue()))
+				metrics.NetCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(netChange)
 			}
 			if resourcename == corev1.ResourceMemory {
 				metrics.AppliedMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.Value()))
+				metrics.NetMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(netChange))
+			}
+			if netChange > 0 {
+				metrics.IncreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1)
+			}
+			if netChange < 0 {
+				metrics.DecreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1)
 			}
 		}
 	}