From fbe48ebcf65e63e1a1ed4522fe708af4d015a634 Mon Sep 17 00:00:00 2001 From: Stavros Kontopoulos Date: Wed, 31 Jul 2024 17:56:07 +0300 Subject: [PATCH] updates --- CONCEPTS-API.md | 133 +++++++++++++++--- .../autoscaling/hpa/helpers/helpers.go | 82 +++++++++++ .../autoscaling/hpa/keda_hpa_test.go | 38 ++--- .../autoscaling/hpa/resources/keda.go | 46 +++++- .../autoscaling/hpa/resources/keda_test.go | 56 ++++++-- .../hpa/resources/scaled_object.go | 25 ++-- .../test_images/metrics-test/service_cpu.yaml | 66 +++++++++ .../metrics-test/service_memory.yaml | 66 +++++++++ .../service_multiple_triggers.yaml | 68 +++++++++ ...e_multiple_triggers_scaling_modifiers.yaml | 70 +++++++++ .../autoscaling/hpa/resources/hpa.go | 118 ---------------- vendor/modules.txt | 1 - 12 files changed, 578 insertions(+), 191 deletions(-) create mode 100644 test/test_images/metrics-test/service_cpu.yaml create mode 100644 test/test_images/metrics-test/service_memory.yaml create mode 100644 test/test_images/metrics-test/service_multiple_triggers.yaml create mode 100644 test/test_images/metrics-test/service_multiple_triggers_scaling_modifiers.yaml delete mode 100644 vendor/knative.dev/serving/pkg/reconciler/autoscaling/hpa/resources/hpa.go diff --git a/CONCEPTS-API.md b/CONCEPTS-API.md index 95b9b9da..0c7140ae 100644 --- a/CONCEPTS-API.md +++ b/CONCEPTS-API.md @@ -2,61 +2,140 @@ The goal of the Autoscaler KEDA extension is to keep the existing UX provided by the current Knative Serving and HPA integration. It builds on top of that to allow more fine-grained configurations and replaces the current HPA implementation with a more flexible and powerful one. -The point of integration between KEDA and this extension is the existance of an HPA resource. +The point of integration between KEDA and this extension is the existence of an HPA resource. If an HPA resource is available for a Knative Service, then the extension can reconcile the corresponding SKS in order networking setup to follow up. In the current implementation at the core Serving, this HPA resource is created by the Serving controller based on the Pod Autoscaler (PA) resource for the current revision. -The latter happens when the user sets the annotation: `autoscaling.knative.dev/class: hpa.autoscaling.knative.dev` in the ksvc. +The latter happens when the user sets the annotation: `autoscaling.knative.dev/class: hpa.autoscaling.knative.dev` in the Knative service (ksvc). In this extension we create a ScaledObject instead of an HPA resource and delegate the HPA creation to KEDA. The above annotation still needs to exist for the extension to pick up the corresponding PA resource created for each revision -and to create the ScaledObject based on that PA resource. -With the approach taken with this extension we can benefit from using KEDA for a shorter monitoring cycle (no need for a Prometheus Adapter service) and it's features eg. multiple triggers. -Users can drive configurations from the ksvc resource via annotations or can bring their own ScaledObject. -The extension supports two modes: ScaledObject auto-creation and bring your own (BYO) ScaledObject. +and to create the ScaledObject based on that PA resource. With this approach we can benefit from using KEDA for a shorter monitoring cycle (no need for a Prometheus Adapter service) and its features eg. multiple triggers. +Users can drive configurations from the ksvc resource via annotations or can bring their own ScaledObject. The extension supports two modes: ScaledObject auto-creation and bring your own (BYO) ScaledObject. # Configuration API -The idea is to configure the ScaledObject resource via annotations. This is done incrementally. Annotations need to be specified in the ksvc resource at the `spec.template.metadata.annotations` level. -A user can start with "cpu" or "memory" for the metric annotation as with current Serving HPA support and need to only specify a target. -The rest is automatically setup by the extension and the corresponding ScaledObject is created. If the user chooses a custom metric then he needs to define the Prometheus address and query through the following annotations: +## Basic configuration + +The idea is to configure the ScaledObject resource via annotations. Annotations need to be specified in the ksvc resource at the `spec.template.metadata.annotations` level. +A user can start with `cpu` or `memory` for the metric annotation as with current Serving HPA support and need to only specify a target. +Note here that if no metric name specified the Serving controller will create a PA resource with a metric set to `cpu` by default. +In any case when defining a `cpu` or `memory` metric the user needs to specify the corresponding `resources.request` for each container deployed. +The rest is automatically setup by the extension and the corresponding ScaledObject is created. + +For example: + +```yaml +... +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/class: "hpa.autoscaling.knative.dev" + autoscaling.knative.dev/metric: "cpu" + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "10" + autoscaling.knative.dev/target: "50" + spec: + containers: + - resources: + requests: + cpu: "100m" +... ``` -autoscaling.knative.dev/prometheus-address: http://prometheus:9090 -autoscaling.knative.dev/prometheus-query: sum(rate(http_requests_total{job="myjob"}[1m])) + +## Custom metric configuration + +If the user chooses a custom metric then he needs to define additionally the metric name, the Prometheus address and the query through the following annotations: + +```yaml +... +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/class: "hpa.autoscaling.knative.dev" + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "10" + autoscaling.knative.dev/target: "5" + autoscaling.knative.dev/metric: "my_metric" + autoscaling.knative.dev/prometheus-address: "http://prometheus:9090" + autoscaling.knative.dev/prometheus-query: sum(rate(http_requests_total{namespace="test"}[1m])) +... ``` -In scenarios where more advanced Prometheus setup are used eg. Thanos, the user can specify the Prometheus auth name, kind and modes. +In scenarios where more advanced Prometheus setup are used e.g. Thanos, the user can specify the Prometheus auth name, kind and modes. + +```yaml +... +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/metric: "my_metric" + autoscaling.knative.dev/prometheus-address: "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092" + autoscaling.knative.dev/prometheus-query: sum(rate(http_requests_total{namespace="test"}[1m])) + autoscaling.knative.dev/trigger-prometheus-auth-name: "keda-trigger-auth-prometheus" + autoscaling.knative.dev/trigger-prometheus-auth-modes: "bearer" + autoscaling.knative.dev/trigger-prometheus-auth-kind: "TriggerAuthentication" +... ``` -autoscaling.knative.dev/prometheus-address: "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092" -autoscaling.knative.dev/trigger-prometheus-auth-name: "keda-trigger-auth-prometheus" -autoscaling.knative.dev/trigger-prometheus-auth-modes: "bearer" -autoscaling.knative.dev/trigger-prometheus-auth-kind: "TriggerAuthentication" + +User can also configure the metric type by defining the following annotation: + ``` +autoscaling.knative.dev/metric-type: "AverageValue" +``` + +Allowed values are: `AverageValue`, `Utilization`, `Value`. +If no metric type is used the following defaults are used: +- cpu: Utilization +- memory: AverageValue +- custom: AverageValue + +Allowed values for the metric type are: +- cpu: Utilization, AverageValue +- memory: Utilization, AverageValue +- custom: Utilization, AverageValue, Value For cpu, memory and a custom metric the extension creates a default Prometheus trigger, however user can add more triggers in json format via the following annotation: ``` -autoscaling.knative.dev/extra-prometheus-triggers="[{"type": "prometheus", "metadata": { "serverAddress": "http://prometheus-operated.default.svc:9090" , "namespace": "test-namespace", "query": "sum(rate(http_requests_total{}[1m]))", "threshold": "5"}}]" +autoscaling.knative.dev/extra-prometheus-triggers: '[{"type": "prometheus", "name": "trigger2", "metadata": { "serverAddress": "http://prometheus-operated.default.svc:9090" , "namespace": "test-namespace", "query": "sum(rate(http_requests_total{}[1m]))", "threshold": "5"}}]' ``` This is useful when the user wants to scale based on multiple metrics and also in combination with the scaling modifiers feature. The scaling modifiers annotation allows to configure that property in the ScaledObject using json format: +```yaml +... +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/metric: "my_metric" + autoscaling.knative.dev/trigger-prometheus-name: "trigger1" + autoscaling.knative.dev/prometheus-query: sum(rate(http_requests_total{namespace="test"}[1m])) + autoscaling.knative.dev/scaling-modifiers: '{"formula": "(trigger1 + trigger2)/2", "target": "5", "activationTarget": "1", "metricType": "AverageValue"}' + autoscaling.knative.dev/extra-prometheus-triggers: '[{"type": "prometheus", "name": "trigger2", "metadata": { "serverAddress": "http://prometheus-operated.default.svc:9090" , "namespace": "test-namespace", "query": "sum(rate(http_requests_total{namespace=\"test\"}[1m]))", "threshold": "5"}}]' +... ``` -autoscaling.knative.dev/scaling-modifiers="{"formula": "(sum(rate(http_requests_total{}[1m])) + sum(rate(http_requests_total{}[1m])))/2", "target": "5", "activationTarget": "1", "metricType": "AverageValue"}" -``` + +The extension creates a default Prometheus trigger named `default-trigger`, here we change the name via the annotation `autoscaling.knative.dev/trigger-prometheus-name`. Then we use that name as part of the formula. +The second trigger is defined completely via an annotation as seen above. + +## Override the ScaledObject The user can also specify the ScaledObject directly in json format via the following annotation: ``` -autoscaling.knative.dev/scaled-object-override = "{...}" +autoscaling.knative.dev/scaled-object-override: '{...}' ``` + The extension will make sure that the ScaledObject is created with proper settings like annotations, max replicas, owner references and name. The rest of the object properties will be passed without modification. This is meant for power users who want to have full control over the ScaledObject but still let the extension manage the target reference. In the scenario where the user wants to bring his own ScaledObject and disable the autoscreation he will have to track revisions -make sure the targetRef is correct each time he brings his own ScaledObject. - -``` +make sure the targetRef is correct each time. If the user does not want the extension to create the ScaledObject, he can bring his own ScaledObject and disable the autoscreation either globally in autoscaler-keda configmap or per ksvc using the annotation: @@ -64,3 +143,11 @@ using the annotation: ``` autoscaling.knative.dev/scaled-object-auto-create: "false" ``` + +## HPA Advanced Configuration + +HPA allows to stabilize the scaling process by introducing a stabilization window. By default, this is 5 minutes. +If user has specified a window annotation for example `autoscaling.knative.dev/window: "20s"` then the extension will pass this setting to the HPA scale up/down +stabilization period. The stabilization window is used by the HPA algorithm to gather recommendations. At the end of it a decision is made. +See [here](https://github.com/kubernetes/enhancements/blob/master/keps/sig-autoscaling/853-configurable-hpa-scale-velocity/README.md) for more. + diff --git a/pkg/reconciler/autoscaling/hpa/helpers/helpers.go b/pkg/reconciler/autoscaling/hpa/helpers/helpers.go index 80afaecf..890fb9fd 100644 --- a/pkg/reconciler/autoscaling/hpa/helpers/helpers.go +++ b/pkg/reconciler/autoscaling/hpa/helpers/helpers.go @@ -17,12 +17,19 @@ limitations under the License. package helpers import ( + "math" "net/url" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/networking/pkg/apis/networking" + "knative.dev/pkg/kmeta" + "knative.dev/pkg/ptr" + "knative.dev/serving/pkg/apis/autoscaling" autoscalingv1alpha1 "knative.dev/serving/pkg/apis/autoscaling/v1alpha1" + "knative.dev/serving/pkg/autoscaler/config/autoscalerconfig" pkgtesting "knative.dev/serving/pkg/testing" //nolint:all ) @@ -74,3 +81,78 @@ func ParseServerAddress(address string) error { } return nil } + +// MakeHPA creates an HPA resource from a PA resource. +func MakeHPA(pa *autoscalingv1alpha1.PodAutoscaler, config *autoscalerconfig.Config) *autoscalingv2.HorizontalPodAutoscaler { + min, max := pa.ScaleBounds(config) + if max == 0 { + max = math.MaxInt32 // default to no limit + } + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: pa.Name, + Namespace: pa.Namespace, + Labels: pa.Labels, + Annotations: pa.Annotations, + OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(pa)}, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: pa.Spec.ScaleTargetRef.APIVersion, + Kind: pa.Spec.ScaleTargetRef.Kind, + Name: pa.Spec.ScaleTargetRef.Name, + }, + }, + } + hpa.Spec.MaxReplicas = max + if min > 0 { + hpa.Spec.MinReplicas = &min + } + + if target, ok := pa.Target(); ok { + switch pa.Metric() { + case autoscaling.CPU: + hpa.Spec.Metrics = []autoscalingv2.MetricSpec{{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.Int32(int32(math.Ceil(target))), + }, + }, + }} + + case autoscaling.Memory: + memory := resource.NewQuantity(int64(target)*1024*1024, resource.BinarySI) + hpa.Spec.Metrics = []autoscalingv2.MetricSpec{{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: memory, + }, + }, + }} + default: + if target, ok := pa.Target(); ok { + targetQuantity := resource.NewQuantity(int64(target), resource.DecimalSI) + hpa.Spec.Metrics = []autoscalingv2.MetricSpec{{ + Type: autoscalingv2.PodsMetricSourceType, + Pods: &autoscalingv2.PodsMetricSource{ + Metric: autoscalingv2.MetricIdentifier{ + Name: pa.Metric(), + }, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: targetQuantity, + }, + }, + }} + } + } + } + + return hpa +} diff --git a/pkg/reconciler/autoscaling/hpa/keda_hpa_test.go b/pkg/reconciler/autoscaling/hpa/keda_hpa_test.go index 9f154d77..8ff7dd84 100644 --- a/pkg/reconciler/autoscaling/hpa/keda_hpa_test.go +++ b/pkg/reconciler/autoscaling/hpa/keda_hpa_test.go @@ -52,7 +52,6 @@ import ( fakepainformer "knative.dev/serving/pkg/client/injection/informers/autoscaling/v1alpha1/podautoscaler/fake" pareconciler "knative.dev/serving/pkg/client/injection/reconciler/autoscaling/v1alpha1/podautoscaler" areconciler "knative.dev/serving/pkg/reconciler/autoscaling" - "knative.dev/serving/pkg/reconciler/autoscaling/hpa/resources" aresources "knative.dev/serving/pkg/reconciler/autoscaling/resources" "knative.dev/serving/pkg/reconciler/serverlessservice/resources/names" @@ -140,7 +139,7 @@ func TestReconcile(t *testing.T) { Objects: []runtime.Object{ helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass), hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, - WithHPAClass, WithMetricAnnotation("cpu"))), + WithHPAClass, WithMetricAnnotation("cpu")), withHPADesiredReplicas(1)), deploy(helpers.TestNamespace, helpers.TestRevision), }, Key: key(helpers.TestNamespace, helpers.TestRevision), @@ -157,23 +156,23 @@ func TestReconcile(t *testing.T) { sks(helpers.TestNamespace, helpers.TestRevision, WithDeployRef(deployName)), }, WantStatusUpdates: []ktesting.UpdateActionImpl{{ - Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(0, 0), + Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(1, 0), WithTraffic, WithPASKSNotReady("SKS Services are not ready yet")), }, { - Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(0, 0), + Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(1, 0), WithTraffic, WithPASKSNotReady("SKS Services are not ready yet")), }}, }, { Name: "reconcile sks is still not ready", Objects: []runtime.Object{ - hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu"))), + hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu")), withHPADesiredReplicas(1)), helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass), deploy(helpers.TestNamespace, helpers.TestRevision), sks(helpers.TestNamespace, helpers.TestRevision, WithDeployRef(deployName), WithPubService, WithPrivateService), }, WantStatusUpdates: []ktesting.UpdateActionImpl{{ - Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithTraffic, withScales(0, 0), + Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithTraffic, withScales(1, 0), WithPASKSNotReady("SKS Services are not ready yet"), WithTraffic, WithPAStatusService(helpers.TestRevision), WithPAMetricsService(privateSvc)), }}, @@ -182,13 +181,13 @@ func TestReconcile(t *testing.T) { Name: "reconcile sks becomes ready, scale target not initialized", Objects: []runtime.Object{ hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithPASKSNotReady("I wasn't ready yet :-("), - WithMetricAnnotation("cpu"))), + WithMetricAnnotation("cpu")), withHPADesiredReplicas(1)), helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithPAStatusService("the-wrong-one")), deploy(helpers.TestNamespace, helpers.TestRevision), sks(helpers.TestNamespace, helpers.TestRevision, WithDeployRef(deployName), WithSKSReady), }, WantStatusUpdates: []ktesting.UpdateActionImpl{{ - Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(0, 0), + Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(1, 0), WithPASKSReady, WithTraffic, WithPAStatusService(helpers.TestRevision), WithPAMetricsService(privateSvc)), }}, @@ -230,14 +229,14 @@ func TestReconcile(t *testing.T) { }, { Name: "reconcile unhappy sks", Objects: []runtime.Object{ - hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu"))), + hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu")), withHPADesiredReplicas(1)), helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithTraffic), deploy(helpers.TestNamespace, helpers.TestRevision), sks(helpers.TestNamespace, helpers.TestRevision, WithDeployRef(deployName+"-hairy"), WithPubService, WithPrivateService), }, WantStatusUpdates: []ktesting.UpdateActionImpl{{ - Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(0, 0), + Object: helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(1, 0), WithPASKSNotReady("SKS Services are not ready yet"), WithTraffic, WithPAStatusService(helpers.TestRevision), WithPAMetricsService(privateSvc)), }}, @@ -249,10 +248,10 @@ func TestReconcile(t *testing.T) { }, { Name: "reconcile sks - update fails", Objects: []runtime.Object{ - helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithTraffic, withScales(0, 0)), + helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithTraffic, withScales(1, 0)), deploy(helpers.TestNamespace, helpers.TestRevision), sks(helpers.TestNamespace, helpers.TestRevision, WithDeployRef("bar"), WithSKSReady), - hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu"))), + hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu")), withHPADesiredReplicas(1)), }, Key: key(helpers.TestNamespace, helpers.TestRevision), WithReactors: []ktesting.ReactionFunc{ @@ -268,9 +267,9 @@ func TestReconcile(t *testing.T) { }, { Name: "create sks - create fails", Objects: []runtime.Object{ - helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(0, 0), WithTraffic), + helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, withScales(1, 0), WithTraffic), deploy(helpers.TestNamespace, helpers.TestRevision), - hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu"))), + hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu")), withHPADesiredReplicas(1)), }, Key: key(helpers.TestNamespace, helpers.TestRevision), WithReactors: []ktesting.ReactionFunc{ @@ -289,7 +288,7 @@ func TestReconcile(t *testing.T) { helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass), deploy(helpers.TestNamespace, helpers.TestRevision), sks(helpers.TestNamespace, helpers.TestRevision, WithDeployRef(deployName), WithSKSOwnersRemoved, WithSKSReady), - hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu"))), + hpa(helpers.PodAutoscaler(helpers.TestNamespace, helpers.TestRevision, WithHPAClass, WithMetricAnnotation("cpu")), withHPADesiredReplicas(1)), }, Key: key(helpers.TestNamespace, helpers.TestRevision), WantErr: true, @@ -404,8 +403,15 @@ func withHPAScaleStatus(d, a int32) hpaOption { } } +func withHPADesiredReplicas(d int32) hpaOption { + return func(hpa *autoscalingv2.HorizontalPodAutoscaler) { + hpa.Status.DesiredReplicas = d + } +} + func hpa(pa *autoscalingv1alpha1.PodAutoscaler, options ...hpaOption) *autoscalingv2.HorizontalPodAutoscaler { - h := resources.MakeHPA(pa, defaultConfig().Autoscaler) + h := helpers.MakeHPA(pa, defaultConfig().Autoscaler) + h.Spec.MinReplicas = ptr.Int32(1) for _, o := range options { o(h) } diff --git a/pkg/reconciler/autoscaling/hpa/resources/keda.go b/pkg/reconciler/autoscaling/hpa/resources/keda.go index 004db23d..40eeb212 100644 --- a/pkg/reconciler/autoscaling/hpa/resources/keda.go +++ b/pkg/reconciler/autoscaling/hpa/resources/keda.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "log" "math" autoscalingv2 "k8s.io/api/autoscaling/v2" @@ -43,8 +44,11 @@ const ( KedaAutoscaleAnnotationPrometheusAuthName = autoscaling.GroupName + "/trigger-prometheus-auth-name" KedaAutoscaleAnnotationPrometheusAuthKind = autoscaling.GroupName + "/trigger-prometheus-auth-kind" KedaAutoscaleAnnotationPrometheusAuthModes = autoscaling.GroupName + "/trigger-prometheus-auth-modes" + KedaAutoscalerAnnnotationPrometheusName = autoscaling.GroupName + "/trigger-prometheus-name" KedaAutoscaleAnnotationExtraPrometheusTriggers = autoscaling.GroupName + "/extra-prometheus-triggers" KedaAutoscaleAnnotationScalingModifiers = autoscaling.GroupName + "/scaling-modifiers" + KedaAutoscalingAnnotationHPAScaleUpRules = autoscaling.GroupName + "/hpa-scale-up-rules" + KedaAutoscalingAnnotationHPAScaleDownRules = autoscaling.GroupName + "/hpa-scale-down-rules" KedaAutoscaleAnnotationsScaledObjectOverride = autoscaling.GroupName + "/scaled-object-override" ) @@ -77,10 +81,13 @@ func DesiredScaledObject(ctx context.Context, pa *autoscalingv1alpha1.PodAutosca return nil, fmt.Errorf("unable to unmarshal scaling modifiers: %w", err) } sO.Spec.Advanced.ScalingModifiers = scalingModifiers + log.Printf("scaling modifiers: %v\n", scalingModifiers) } if min > 0 { sO.Spec.MinReplicaCount = ptr.Int32(min) + } else { + sO.Spec.MinReplicaCount = ptr.Int32(1) } if target, ok := pa.Target(); ok { @@ -92,6 +99,7 @@ func DesiredScaledObject(ctx context.Context, pa *autoscalingv1alpha1.PodAutosca case autoscaling.CPU: sO.Spec.Triggers = []v1alpha1.ScaleTriggers{ { + Name: "default-trigger-cpu", Type: "cpu", MetricType: *mt, Metadata: map[string]string{"value": fmt.Sprint(int32(math.Ceil(target)))}, @@ -104,6 +112,7 @@ func DesiredScaledObject(ctx context.Context, pa *autoscalingv1alpha1.PodAutosca memory := resource.NewQuantity(int64(target)*1024*1024, resource.BinarySI) sO.Spec.Triggers = []v1alpha1.ScaleTriggers{ { + Name: "default-trigger-memory", Type: "memory", MetricType: *mt, Metadata: map[string]string{"value": memory.String()}, @@ -115,11 +124,10 @@ func DesiredScaledObject(ctx context.Context, pa *autoscalingv1alpha1.PodAutosca default: targetQuantity := resource.NewQuantity(int64(target), resource.DecimalSI) var query, address string - if v, ok := pa.Annotations[KedaAutoscaleAnnotationPrometheusQuery]; ok { - query = v - } else { - query = fmt.Sprintf("sum(rate(%s{}[1m]))", pa.Metric()) + if query, ok = pa.Annotations[KedaAutoscaleAnnotationPrometheusQuery]; !ok { + return nil, fmt.Errorf("query is missing for custom metric: %w", err) } + if v, ok := pa.Annotations[KedaAutoscaleAnnotationPrometheusAddress]; ok { if err := helpers.ParseServerAddress(v); err != nil { return nil, fmt.Errorf("invalid prometheus address: %w", err) @@ -155,12 +163,42 @@ func DesiredScaledObject(ctx context.Context, pa *autoscalingv1alpha1.PodAutosca } } + if v, ok := pa.Annotations[KedaAutoscalingAnnotationHPAScaleUpRules]; ok { + var scaleUpRules autoscalingv2.HPAScalingRules + if err := json.Unmarshal([]byte(v), &scaleUpRules); err != nil { + return nil, fmt.Errorf("unable to unmarshal scale up rules: %w", err) + } + if sO.Spec.Advanced.HorizontalPodAutoscalerConfig.Behavior == nil { + sO.Spec.Advanced.HorizontalPodAutoscalerConfig.Behavior = &autoscalingv2.HorizontalPodAutoscalerBehavior{} + } + sO.Spec.Advanced.HorizontalPodAutoscalerConfig.Behavior.ScaleUp = &scaleUpRules + } + + if v, ok := pa.Annotations[KedaAutoscalingAnnotationHPAScaleDownRules]; ok { + var scaleDownRules autoscalingv2.HPAScalingRules + if err := json.Unmarshal([]byte(v), &scaleDownRules); err != nil { + return nil, fmt.Errorf("unable to unmarshal scale down rules: %w", err) + } + if sO.Spec.Advanced.HorizontalPodAutoscalerConfig.Behavior == nil { + sO.Spec.Advanced.HorizontalPodAutoscalerConfig.Behavior = &autoscalingv2.HorizontalPodAutoscalerBehavior{} + } + sO.Spec.Advanced.HorizontalPodAutoscalerConfig.Behavior.ScaleDown = &scaleDownRules + } + return &sO, nil } func getDefaultPrometheusTrigger(annotations map[string]string, address string, query string, threshold string, ns string, targetType autoscalingv2.MetricTargetType) (*v1alpha1.ScaleTriggers, error) { + var name string + + if v, ok := annotations[KedaAutoscalerAnnnotationPrometheusName]; ok { + name = v + } else { + name = "default-trigger-custom" + } trigger := v1alpha1.ScaleTriggers{ Type: "prometheus", + Name: name, MetricType: targetType, Metadata: map[string]string{ "serverAddress": address, diff --git a/pkg/reconciler/autoscaling/hpa/resources/keda_test.go b/pkg/reconciler/autoscaling/hpa/resources/keda_test.go index 89ce2a1e..1b84f01a 100644 --- a/pkg/reconciler/autoscaling/hpa/resources/keda_test.go +++ b/pkg/reconciler/autoscaling/hpa/resources/keda_test.go @@ -49,8 +49,8 @@ func TestDesiredScaledObject(t *testing.T) { Autoscaler: aConfig, AutoscalerKeda: autoscalerKedaConfig}) - extraTrigger := fmt.Sprintf("[{\"type\": \"prometheus\", \"metadata\": { \"serverAddress\": \"%s\" , \"namespace\": \"%s\", \"query\": \"sum(rate(http_requests_total{}[1m]))\", \"threshold\": \"5\"}}]", hpaconfig.DefaultPrometheusAddress, helpers.TestNamespace) - scalingModifiers := `{"formula": "(sum(rate(http_requests_total{}[1m])) + sum(rate(http_requests_total{}[1m])))/2", "target": "5", "activationTarget": "1", "metricType": "AverageValue"}` + extraTrigger := fmt.Sprintf("[{ \"name\": \"trigger2\", \"type\": \"prometheus\", \"metadata\": { \"serverAddress\": \"%s\" , \"namespace\": \"%s\", \"query\": \"sum(rate(http_requests_total{}[1m]))\", \"threshold\": \"5\"}}]", hpaconfig.DefaultPrometheusAddress, helpers.TestNamespace) + scalingModifiers := `{"formula": "(trigger1 + trigger2)/2", "target": "5", "activationTarget": "1", "metricType": "AverageValue"}` scaledObjectTests := []struct { name string @@ -71,7 +71,7 @@ func TestDesiredScaledObject(t *testing.T) { autoscaling.TargetAnnotationKey: "75", autoscaling.ClassAnnotationKey: autoscaling.HPA, }), WithMaxScale(10), WithMinScale(1), WithScaleTargetRef(helpers.TestRevision+"-deployment"), - WithTrigger("cpu", autoscalingv2.UtilizationMetricType, map[string]string{ + WithTrigger("default-trigger-cpu", "cpu", autoscalingv2.UtilizationMetricType, map[string]string{ "value": "75", }), WithHorizontalPodAutoscalerConfig(helpers.TestRevision)), }, { @@ -108,7 +108,7 @@ func TestDesiredScaledObject(t *testing.T) { autoscaling.TargetAnnotationKey: "75", autoscaling.ClassAnnotationKey: autoscaling.HPA, }), WithMaxScale(10), WithMinScale(2), WithScaleTargetRef(helpers.TestRevision+"-deployment"), - WithTrigger("cpu", autoscalingv2.UtilizationMetricType, map[string]string{ + WithTrigger("default-trigger-cpu", "cpu", autoscalingv2.UtilizationMetricType, map[string]string{ "value": "75", }), WithHorizontalPodAutoscalerConfig(helpers.TestRevision)), }, { @@ -137,7 +137,7 @@ func TestDesiredScaledObject(t *testing.T) { autoscaling.TargetAnnotationKey: "200", autoscaling.ClassAnnotationKey: autoscaling.HPA, }), WithMaxScale(10), WithMinScale(1), WithScaleTargetRef(helpers.TestRevision+"-deployment"), - WithTrigger("memory", autoscalingv2.AverageValueMetricType, map[string]string{ + WithTrigger("default-trigger-memory", "memory", autoscalingv2.AverageValueMetricType, map[string]string{ "value": "200Mi", }), WithHorizontalPodAutoscalerConfig(helpers.TestRevision)), }, { @@ -157,7 +157,7 @@ func TestDesiredScaledObject(t *testing.T) { KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", autoscaling.TargetAnnotationKey: "5", autoscaling.ClassAnnotationKey: autoscaling.HPA, - }), WithMaxScale(10), WithMinScale(1), WithTrigger("prometheus", autoscalingv2.AverageValueMetricType, map[string]string{ + }), WithMaxScale(10), WithMinScale(1), WithTrigger("default-trigger-custom", "prometheus", autoscalingv2.AverageValueMetricType, map[string]string{ "namespace": helpers.TestNamespace, "query": "sum(rate(http_requests_total{}[1m]))", "threshold": "5", @@ -180,7 +180,7 @@ func TestDesiredScaledObject(t *testing.T) { autoscaling.MetricAnnotationKey: "cpu", autoscaling.TargetAnnotationKey: "50", autoscaling.ClassAnnotationKey: autoscaling.HPA, - }), WithMaxScale(10), WithMinScale(1), WithCpuTrigger(map[string]string{"value": "50"}), WithPrometheusTrigger(map[string]string{ + }), WithMaxScale(10), WithMinScale(1), WithCpuTrigger(map[string]string{"value": "50"}), WithTrigger("trigger2", "prometheus", "", map[string]string{ "namespace": helpers.TestNamespace, "query": "sum(rate(http_requests_total{}[1m]))", "threshold": "5", @@ -191,26 +191,36 @@ func TestDesiredScaledObject(t *testing.T) { paAnnotations: map[string]string{ autoscaling.MinScaleAnnotationKey: "1", autoscaling.MaxScaleAnnotationKey: "10", - autoscaling.MetricAnnotationKey: "cpu", - autoscaling.TargetAnnotationKey: "50", + autoscaling.MetricAnnotationKey: "metric", + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + autoscaling.TargetAnnotationKey: "5", + KedaAutoscalerAnnnotationPrometheusName: "trigger1", KedaAutoscaleAnnotationExtraPrometheusTriggers: extraTrigger, KedaAutoscaleAnnotationScalingModifiers: scalingModifiers, }, wantScaledObject: ScaledObject(helpers.TestNamespace, helpers.TestRevision, WithAnnotations(map[string]string{ KedaAutoscaleAnnotationExtraPrometheusTriggers: extraTrigger, + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", autoscaling.MinScaleAnnotationKey: "1", autoscaling.MaxScaleAnnotationKey: "10", - autoscaling.MetricAnnotationKey: "cpu", - autoscaling.TargetAnnotationKey: "50", + autoscaling.MetricAnnotationKey: "metric", + autoscaling.TargetAnnotationKey: "5", autoscaling.ClassAnnotationKey: autoscaling.HPA, + KedaAutoscalerAnnnotationPrometheusName: "trigger1", KedaAutoscaleAnnotationScalingModifiers: scalingModifiers, }), WithScaleTargetRef(helpers.TestRevision+"-deployment"), WithScalingModifiers(kedav1alpha1.ScalingModifiers{ - Formula: "(sum(rate(http_requests_total{}[1m])) + sum(rate(http_requests_total{}[1m])))/2", + Formula: "(trigger1 + trigger2)/2", Target: "5", ActivationTarget: "1", MetricType: "AverageValue", - }), WithMaxScale(10), WithMinScale(1), WithCpuTrigger(map[string]string{"value": "50"}), WithPrometheusTrigger(map[string]string{ + }), WithMaxScale(10), WithMinScale(1), WithTrigger("trigger1", "prometheus", "AverageValue", map[string]string{ + "namespace": helpers.TestNamespace, + "query": "sum(rate(http_requests_total{}[1m]))", + "threshold": "5", + "serverAddress": "http://prometheus-operated.default.svc:9090", + }), + WithTrigger("trigger2", "prometheus", "", map[string]string{ "namespace": helpers.TestNamespace, "query": "sum(rate(http_requests_total{}[1m]))", "threshold": "5", @@ -227,6 +237,16 @@ func TestDesiredScaledObject(t *testing.T) { autoscaling.TargetAnnotationKey: "5", }, wantErr: true, + }, { + name: "custom metric with missing prometheus query", + paAnnotations: map[string]string{ + autoscaling.MinScaleAnnotationKey: "1", + autoscaling.MaxScaleAnnotationKey: "10", + autoscaling.MetricAnnotationKey: "http_requests_total", + KedaAutoscaleAnnotationPrometheusAddress: "http://prometheus-operated.default.svc:9090", + autoscaling.TargetAnnotationKey: "5", + }, + wantErr: true, }, { name: "custom metric with bad auth kind", paAnnotations: map[string]string{ @@ -250,6 +270,16 @@ func TestDesiredScaledObject(t *testing.T) { autoscaling.TargetAnnotationKey: "5", }, wantErr: true, + }, { + name: "custom metric with bad scaled object override", + paAnnotations: map[string]string{ + autoscaling.MinScaleAnnotationKey: "1", + autoscaling.MaxScaleAnnotationKey: "10", + autoscaling.MetricAnnotationKey: "http_requests_total", + KedaAutoscaleAnnotationsScaledObjectOverride: "{bad}", + autoscaling.TargetAnnotationKey: "5", + }, + wantErr: true, }, { name: "custom metric with default cm values with authentication", paAnnotations: map[string]string{ diff --git a/pkg/reconciler/autoscaling/hpa/resources/scaled_object.go b/pkg/reconciler/autoscaling/hpa/resources/scaled_object.go index 9857b3cb..f1eed9bc 100644 --- a/pkg/reconciler/autoscaling/hpa/resources/scaled_object.go +++ b/pkg/reconciler/autoscaling/hpa/resources/scaled_object.go @@ -91,18 +91,10 @@ func WithScalingModifiers(sm kedav1alpha1.ScalingModifiers) ScaledObjectOption { } } -func WithPrometheusTrigger(metadata map[string]string) ScaledObjectOption { - return func(scaledObject *kedav1alpha1.ScaledObject) { - scaledObject.Spec.Triggers = append(scaledObject.Spec.Triggers, kedav1alpha1.ScaleTriggers{ - Type: "prometheus", - Metadata: metadata, - }) - } -} - func WithCpuTrigger(metadata map[string]string) ScaledObjectOption { return func(scaledObject *kedav1alpha1.ScaledObject) { scaledObject.Spec.Triggers = append(scaledObject.Spec.Triggers, kedav1alpha1.ScaleTriggers{ + Name: "default-trigger-cpu", Type: "cpu", Metadata: metadata, MetricType: "Utilization", @@ -114,6 +106,7 @@ func WithAuthTrigger(triggerType string, metricType autoscalingv2.MetricTargetTy return func(scaledObject *kedav1alpha1.ScaledObject) { scaledObject.Spec.Triggers = []kedav1alpha1.ScaleTriggers{ { + Name: "default-trigger-custom", Type: triggerType, MetricType: metricType, Metadata: metadata, @@ -151,13 +144,13 @@ func WithAuthenticationRef(name, kind string) ScaledObjectOption { } } -func WithTrigger(triggerType string, metricType autoscalingv2.MetricTargetType, metadata map[string]string) ScaledObjectOption { +func WithTrigger(name string, triggerType string, metricType autoscalingv2.MetricTargetType, metadata map[string]string) ScaledObjectOption { return func(scaledObject *kedav1alpha1.ScaledObject) { - scaledObject.Spec.Triggers = []kedav1alpha1.ScaleTriggers{ - { - Type: triggerType, - Metadata: metadata, - MetricType: metricType, - }} + scaledObject.Spec.Triggers = append(scaledObject.Spec.Triggers, kedav1alpha1.ScaleTriggers{ + Name: name, + Type: triggerType, + Metadata: metadata, + MetricType: metricType, + }) } } diff --git a/test/test_images/metrics-test/service_cpu.yaml b/test/test_images/metrics-test/service_cpu.yaml new file mode 100644 index 00000000..d56c06e8 --- /dev/null +++ b/test/test_images/metrics-test/service_cpu.yaml @@ -0,0 +1,66 @@ +# Copyright 2024 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: metrics-test +spec: + template: + metadata: + labels: + app: metrics-test + annotations: + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "10" + autoscaling.knative.dev/target: "20" + autoscaling.knative.dev/class: "hpa.autoscaling.knative.dev" + autoscaling.knative.dev/metric: "cpu" + spec: + containers: + - image: ko://knative.dev/autoscaler-keda/test/test_images/metrics-test/ + imagePullPolicy: Always + ports: + - name: http1 + containerPort: 8080 +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + name: metrics-test-sm +spec: + endpoints: + - port: metrics + scheme: http + namespaceSelector: {} + selector: + matchLabels: + name: metrics-test-sm +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: metrics-test-sm + name: metrics-test-sm +spec: + ports: + - name: metrics + port: 9096 + protocol: TCP + targetPort: 9096 + selector: + serving.knative.dev/service: metrics-test + type: ClusterIP diff --git a/test/test_images/metrics-test/service_memory.yaml b/test/test_images/metrics-test/service_memory.yaml new file mode 100644 index 00000000..ce1bece0 --- /dev/null +++ b/test/test_images/metrics-test/service_memory.yaml @@ -0,0 +1,66 @@ +# Copyright 2024 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: metrics-test +spec: + template: + metadata: + labels: + app: metrics-test + annotations: + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "10" + autoscaling.knative.dev/target: "70" + autoscaling.knative.dev/class: "hpa.autoscaling.knative.dev" + autoscaling.knative.dev/metric: "memory" + spec: + containers: + - image: ko://knative.dev/autoscaler-keda/test/test_images/metrics-test/ + imagePullPolicy: Always + ports: + - name: http1 + containerPort: 8080 +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + name: metrics-test-sm +spec: + endpoints: + - port: metrics + scheme: http + namespaceSelector: {} + selector: + matchLabels: + name: metrics-test-sm +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: metrics-test-sm + name: metrics-test-sm +spec: + ports: + - name: metrics + port: 9096 + protocol: TCP + targetPort: 9096 + selector: + serving.knative.dev/service: metrics-test + type: ClusterIP diff --git a/test/test_images/metrics-test/service_multiple_triggers.yaml b/test/test_images/metrics-test/service_multiple_triggers.yaml new file mode 100644 index 00000000..61f75d68 --- /dev/null +++ b/test/test_images/metrics-test/service_multiple_triggers.yaml @@ -0,0 +1,68 @@ +# Copyright 2024 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: metrics-test +spec: + template: + metadata: + labels: + app: metrics-test + annotations: + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "10" + autoscaling.knative.dev/target: "1" + autoscaling.knative.dev/class: "hpa.autoscaling.knative.dev" + autoscaling.knative.dev/metric: "my_metric" + autoscaling.knative.dev/prometheus-query: "sum(rate(http_requests_total{namespace=\"test\"}[1m]))" + autoscaling.knative.dev/extra-prometheus-triggers: '[{"type": "prometheus", "name": "extra-trigger", "metadata": { "serverAddress": "http://prometheus-operated.default.svc:9090" , "namespace": "test-namespace", "query": "sum(rate(http_requests_total{namespace=\"test\"}[1m]))", "threshold": "1"}}]' + spec: + containers: + - image: ko://knative.dev/autoscaler-keda/test/test_images/metrics-test/ + imagePullPolicy: Always + ports: + - name: http1 + containerPort: 8080 +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + name: metrics-test-sm +spec: + endpoints: + - port: metrics + scheme: http + namespaceSelector: {} + selector: + matchLabels: + name: metrics-test-sm +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: metrics-test-sm + name: metrics-test-sm +spec: + ports: + - name: metrics + port: 9096 + protocol: TCP + targetPort: 9096 + selector: + serving.knative.dev/service: metrics-test + type: ClusterIP diff --git a/test/test_images/metrics-test/service_multiple_triggers_scaling_modifiers.yaml b/test/test_images/metrics-test/service_multiple_triggers_scaling_modifiers.yaml new file mode 100644 index 00000000..55dd8bc7 --- /dev/null +++ b/test/test_images/metrics-test/service_multiple_triggers_scaling_modifiers.yaml @@ -0,0 +1,70 @@ +# Copyright 2024 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: metrics-test +spec: + template: + metadata: + labels: + app: metrics-test + annotations: + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "10" + autoscaling.knative.dev/target: "1" + autoscaling.knative.dev/class: "hpa.autoscaling.knative.dev" + autoscaling.knative.dev/metric: "my_metric" + autoscaling.knative.dev/trigger-prometheus-name: "trigger1" + autoscaling.knative.dev/prometheus-query: sum(rate(http_requests_total{namespace="test"}[1m])) + autoscaling.knative.dev/scaling-modifiers: '{"formula": "(trigger1 + trigger2)/2", "target": "1", "activationTarget": "1", "metricType": "AverageValue"}' + autoscaling.knative.dev/extra-prometheus-triggers: '[{"type": "prometheus", "name": "trigger2", "metadata": { "serverAddress": "http://prometheus-operated.default.svc:9090" , "namespace": "test-namespace", "query": "sum(rate(http_requests_total{namespace=\"test\"}[1m]))", "threshold": "1"}}]' + spec: + containers: + - image: ko://knative.dev/autoscaler-keda/test/test_images/metrics-test/ + imagePullPolicy: Always + ports: + - name: http1 + containerPort: 8080 +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + name: metrics-test-sm +spec: + endpoints: + - port: metrics + scheme: http + namespaceSelector: {} + selector: + matchLabels: + name: metrics-test-sm +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: metrics-test-sm + name: metrics-test-sm +spec: + ports: + - name: metrics + port: 9096 + protocol: TCP + targetPort: 9096 + selector: + serving.knative.dev/service: metrics-test + type: ClusterIP diff --git a/vendor/knative.dev/serving/pkg/reconciler/autoscaling/hpa/resources/hpa.go b/vendor/knative.dev/serving/pkg/reconciler/autoscaling/hpa/resources/hpa.go deleted file mode 100644 index ebdc276a..00000000 --- a/vendor/knative.dev/serving/pkg/reconciler/autoscaling/hpa/resources/hpa.go +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright 2018 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "math" - - autoscalingv2 "k8s.io/api/autoscaling/v2" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "knative.dev/pkg/kmeta" - "knative.dev/pkg/ptr" - - "knative.dev/serving/pkg/apis/autoscaling" - autoscalingv1alpha1 "knative.dev/serving/pkg/apis/autoscaling/v1alpha1" - "knative.dev/serving/pkg/autoscaler/config/autoscalerconfig" -) - -// MakeHPA creates an HPA resource from a PA resource. -func MakeHPA(pa *autoscalingv1alpha1.PodAutoscaler, config *autoscalerconfig.Config) *autoscalingv2.HorizontalPodAutoscaler { - min, max := pa.ScaleBounds(config) - if max == 0 { - max = math.MaxInt32 // default to no limit - } - hpa := &autoscalingv2.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Name: pa.Name, - Namespace: pa.Namespace, - Labels: pa.Labels, - Annotations: pa.Annotations, - OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(pa)}, - }, - Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ - APIVersion: pa.Spec.ScaleTargetRef.APIVersion, - Kind: pa.Spec.ScaleTargetRef.Kind, - Name: pa.Spec.ScaleTargetRef.Name, - }, - }, - } - hpa.Spec.MaxReplicas = max - if min > 0 { - hpa.Spec.MinReplicas = &min - } - - if target, ok := pa.Target(); ok { - switch pa.Metric() { - case autoscaling.CPU: - hpa.Spec.Metrics = []autoscalingv2.MetricSpec{{ - Type: autoscalingv2.ResourceMetricSourceType, - Resource: &autoscalingv2.ResourceMetricSource{ - Name: corev1.ResourceCPU, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.UtilizationMetricType, - AverageUtilization: ptr.Int32(int32(math.Ceil(target))), - }, - }, - }} - - case autoscaling.Memory: - memory := resource.NewQuantity(int64(target)*1024*1024, resource.BinarySI) - hpa.Spec.Metrics = []autoscalingv2.MetricSpec{{ - Type: autoscalingv2.ResourceMetricSourceType, - Resource: &autoscalingv2.ResourceMetricSource{ - Name: corev1.ResourceMemory, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.AverageValueMetricType, - AverageValue: memory, - }, - }, - }} - default: - targetQuantity := resource.NewQuantity(int64(target), resource.DecimalSI) - hpa.Spec.Metrics = []autoscalingv2.MetricSpec{{ - Type: autoscalingv2.PodsMetricSourceType, - Pods: &autoscalingv2.PodsMetricSource{ - Metric: autoscalingv2.MetricIdentifier{ - Name: pa.Metric(), - }, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.AverageValueMetricType, - AverageValue: targetQuantity, - }, - }, - }} - } - } - - if window, hasWindow := pa.Window(); hasWindow { - windowSeconds := int32(window.Seconds()) - hpa.Spec.Behavior = &autoscalingv2.HorizontalPodAutoscalerBehavior{ - ScaleDown: &autoscalingv2.HPAScalingRules{ - StabilizationWindowSeconds: &windowSeconds, - }, - ScaleUp: &autoscalingv2.HPAScalingRules{ - StabilizationWindowSeconds: &windowSeconds, - }, - } - } - - return hpa -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 04ff17dd..46cbf2e7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1097,7 +1097,6 @@ knative.dev/serving/pkg/gc knative.dev/serving/pkg/networking knative.dev/serving/pkg/reconciler/autoscaling knative.dev/serving/pkg/reconciler/autoscaling/config -knative.dev/serving/pkg/reconciler/autoscaling/hpa/resources knative.dev/serving/pkg/reconciler/autoscaling/resources knative.dev/serving/pkg/reconciler/autoscaling/resources/names knative.dev/serving/pkg/reconciler/revision/resources/names