From 8abc7c7ba711e8b6795a5834deda64baa0fbc103 Mon Sep 17 00:00:00 2001 From: Stavros Kontopoulos Date: Tue, 23 Jul 2024 13:45:53 +0300 Subject: [PATCH] BYO scaledobject (#31) * BYO scaledobject * update docs * fix spelling etc --- DEVELOPMENT.md | 38 ++++++-- config/config.yaml | 8 ++ .../autoscaling/hpa/config/autoscaler_keda.go | 7 +- pkg/reconciler/autoscaling/hpa/keda_hpa.go | 72 +++++++++----- .../autoscaling/hpa/resources/keda.go | 20 ++-- .../autoscaling/hpa/resources/keda_test.go | 96 +++++++++---------- test/e2e/autoscale_custom_test.go | 14 +-- 7 files changed, 158 insertions(+), 97 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d87cdc03..5bcd86c4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -169,10 +169,36 @@ prometheus-prometheus-kube-prometheus-prometheus-0 2/2 Running 0 prometheus-prometheus-node-exporter-q5qzv 1/1 Running 0 126m ``` -## Roadmap +### Bring Your Own ScaledObject + +You can bring your own ScaledObject by either disabling globally the auto-creation mode or by setting the `autoscaling.knative.dev/scaled-object-auto-create` annotation to `false` in the service (`spec.template.metadata` field). + +Then at any time you can create the ScaledObject with the desired configuration as follows: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + annotations: + "autoscaling.knative.dev/class": "hpa.autoscaling.knative.dev" + name: metrics-test-00001 + namespace: test +spec: + advanced: + horizontalPodAutoscalerConfig: + name: metrics-test-00001 + scalingModifiers: {} + maxReplicaCount: 10 + minReplicaCount: 1 + scaleTargetRef: + name: metrics-test-00001-deployment + triggers: + - metadata: + namespace: test + query: sum(rate(http_requests_total{namespace="test"}[1m])) + serverAddress: http://prometheus-operated.default.svc:9090 + threshold: "5" + type: prometheus +``` -- Support more functionality wrt hpa configuration compared to the original autoscaler-hpa -- Add e2e tests (Kind) -- Allow user to specify a ScaledObject instead of auto-creating it (non managed mode). Useful also for the KServe integration. -- HA support -- OCP Instructions +The annotation for the scaling class is required so reconciliation is triggered for the HPA that is generated by KEDA. diff --git a/config/config.yaml b/config/config.yaml index 22e0ae03..6bbabdfa 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -40,3 +40,11 @@ data: # configures the default Prometheus address if no is specified per service autoscaler.keda.prometheus-address: "http://prometheus-operated.default.svc:9090" + + # configures the globally default mode for creating ScaledObject. Default is true. + # If you set this to false make sure that you have a ScaledObject for each service + # and existing services with a previously automatically created ScaledObject should + # have their Knative service annotation `autoscaling.knative.dev/scaled-object-auto-create` set to true. + # By setting `autoscaling.knative.dev/scaled-object-auto-create` at the Knative Service level you can bypass + # this configuration and by setting to false you can bring your own scaled object. + autoscaler.keda.scaledobject-autocreate: "true" diff --git a/pkg/reconciler/autoscaling/hpa/config/autoscaler_keda.go b/pkg/reconciler/autoscaling/hpa/config/autoscaler_keda.go index 206509a7..94ec4e6b 100644 --- a/pkg/reconciler/autoscaling/hpa/config/autoscaler_keda.go +++ b/pkg/reconciler/autoscaling/hpa/config/autoscaler_keda.go @@ -34,16 +34,19 @@ const ( // AutoscalerKedaConfig contains autoscaler keda related configuration defined in the // `config-autoscaler-keda` config map. type AutoscalerKedaConfig struct { - PrometheusAddress string + PrometheusAddress string + ShouldCreateScaledObject bool } // NewAutoscalerKedaConfigFromConfigMap creates an AutoscalerKedaConfig from the supplied ConfigMap func NewConfigFromMap(data map[string]string) (*AutoscalerKedaConfig, error) { config := &AutoscalerKedaConfig{ - PrometheusAddress: DefaultPrometheusAddress, + PrometheusAddress: DefaultPrometheusAddress, + ShouldCreateScaledObject: true, } if err := cm.Parse(data, cm.AsString("autoscaler.keda.prometheus-address", &config.PrometheusAddress), + cm.AsBool("autoscaler.keda.scaledobject-autocreate", &config.ShouldCreateScaledObject), ); err != nil { return nil, fmt.Errorf("failed to parse data: %w", err) } diff --git a/pkg/reconciler/autoscaling/hpa/keda_hpa.go b/pkg/reconciler/autoscaling/hpa/keda_hpa.go index 8505718a..b4e5daca 100644 --- a/pkg/reconciler/autoscaling/hpa/keda_hpa.go +++ b/pkg/reconciler/autoscaling/hpa/keda_hpa.go @@ -19,6 +19,10 @@ package hpa import ( "context" "fmt" + "strconv" + + "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "knative.dev/serving/pkg/apis/autoscaling" v2 "k8s.io/api/autoscaling/v2" "k8s.io/apimachinery/pkg/api/equality" @@ -42,7 +46,10 @@ import ( "knative.dev/autoscaler-keda/pkg/reconciler/autoscaling/hpa/resources" ) -const allActivators = 0 +const ( + allActivators = 0 + KedaAutoscaleAnnotationAutocreate = autoscaling.GroupName + "/scaled-object-auto-create" +) // Reconciler implements the control loop for the HPA resources. type Reconciler struct { @@ -64,41 +71,58 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, pa *autoscalingv1alpha1. logger := logging.FromContext(ctx) + var scaledObj *v1alpha1.ScaledObject var hpa *v2.HorizontalPodAutoscaler + shouldCreateScaledObject := true - dScaledObject, err := resources.DesiredScaledObject(pa, hpaconfig.FromContext(ctx).Autoscaler, hpaconfig.FromContext(ctx).AutoscalerKeda) - if err != nil { - return fmt.Errorf("failed to contruct desiredScaledObject: %w", err) + if hpaconfig.FromContext(ctx).AutoscalerKeda != nil { + shouldCreateScaledObject = hpaconfig.FromContext(ctx).AutoscalerKeda.ShouldCreateScaledObject } - scaledObj, err := c.kedaLister.ScaledObjects(pa.Namespace).Get(dScaledObject.Name) - if errors.IsNotFound(err) { - logger.Infof("Creating Scaled Object %q", dScaledObject.Name) - if scaledObj, err = c.kedaClient.KedaV1alpha1().ScaledObjects(dScaledObject.Namespace).Create(ctx, dScaledObject, metav1.CreateOptions{}); err != nil { - pa.Status.MarkResourceFailedCreation("ScaledObject", dScaledObject.Name) - return fmt.Errorf("failed to create ScaledObject: %w", err) + if v, ok := pa.Annotations[KedaAutoscaleAnnotationAutocreate]; ok { + if b, err := strconv.ParseBool(v); err == nil { + shouldCreateScaledObject = b + } else { + logger.Warnf("Failed to parse annotation %q value %q as boolean: %v", KedaAutoscaleAnnotationAutocreate, v, err) } - } else if err != nil { - return fmt.Errorf("failed to get ScaledObject: %w", err) - } else if !metav1.IsControlledBy(scaledObj, pa) { - // Surface an error in the PodAutoscaler's status, and return an error. - pa.Status.MarkResourceNotOwned("ScaledObject", dScaledObject.Name) - return fmt.Errorf("PodAutoscaler: %q does not own ScaledObject: %q", pa.Name, dScaledObject.Name) } - if !equality.Semantic.DeepEqual(dScaledObject.Spec, scaledObj.Spec) { - logger.Infof("Updating ScaledObject %q", dScaledObject.Name) - update := scaledObj.DeepCopy() - update.Spec = dScaledObject.Spec - if _, err := c.kedaClient.KedaV1alpha1().ScaledObjects(pa.Namespace).Update(ctx, update, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("failed to update ScaledObject: %w", err) + + if shouldCreateScaledObject { + dScaledObject, err := resources.DesiredScaledObject(pa, hpaconfig.FromContext(ctx).Autoscaler, hpaconfig.FromContext(ctx).AutoscalerKeda) + if err != nil { + return fmt.Errorf("failed to contruct desiredScaledObject: %w", err) + } + + scaledObj, err = c.kedaLister.ScaledObjects(pa.Namespace).Get(dScaledObject.Name) + if errors.IsNotFound(err) { + logger.Infof("Creating Scaled Object %q", dScaledObject.Name) + if scaledObj, err = c.kedaClient.KedaV1alpha1().ScaledObjects(dScaledObject.Namespace).Create(ctx, dScaledObject, metav1.CreateOptions{}); err != nil { + pa.Status.MarkResourceFailedCreation("ScaledObject", dScaledObject.Name) + return fmt.Errorf("failed to create ScaledObject: %w", err) + } + } else if err != nil { + return fmt.Errorf("failed to get ScaledObject: %w", err) + } else if !metav1.IsControlledBy(scaledObj, pa) { + // Surface an error in the PodAutoscaler's status, and return an error. + pa.Status.MarkResourceNotOwned("ScaledObject", dScaledObject.Name) + return fmt.Errorf("PodAutoscaler: %q does not own ScaledObject: %q", pa.Name, dScaledObject.Name) + } + if !equality.Semantic.DeepEqual(dScaledObject.Spec, scaledObj.Spec) { + logger.Infof("Updating ScaledObject %q", dScaledObject.Name) + update := scaledObj.DeepCopy() + update.Spec = dScaledObject.Spec + if _, err := c.kedaClient.KedaV1alpha1().ScaledObjects(pa.Namespace).Update(ctx, update, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update ScaledObject: %w", err) + } } } - hpa, err = c.hpaLister.HorizontalPodAutoscalers(pa.Namespace).Get(pa.Name) + hpa, err := c.hpaLister.HorizontalPodAutoscalers(pa.Namespace).Get(pa.Name) if errors.IsNotFound(err) { + logger.Infof("Skipping HPA %q", pa.Name) return nil // skip wait to be triggered by hpa events eg. creation } - if scaledObj.Spec.MinReplicaCount != nil { + if scaledObj != nil && scaledObj.Spec.MinReplicaCount != nil { if hpa.Status.DesiredReplicas < *scaledObj.Spec.MinReplicaCount { return nil // skip wait to be triggered by hpa events } diff --git a/pkg/reconciler/autoscaling/hpa/resources/keda.go b/pkg/reconciler/autoscaling/hpa/resources/keda.go index 5d03a391..3bfe8f28 100644 --- a/pkg/reconciler/autoscaling/hpa/resources/keda.go +++ b/pkg/reconciler/autoscaling/hpa/resources/keda.go @@ -37,11 +37,11 @@ import ( ) const ( - KedaAutoscaleAnotationPrometheusAddress = autoscaling.GroupName + "/prometheus-address" - KedaAutoscaleAnotationPrometheusQuery = autoscaling.GroupName + "/prometheus-query" - KedaAutoscaleAnotationPrometheusAuthName = autoscaling.GroupName + "/trigger-prometheus-auth-name" - KedaAutoscaleAnotationPrometheusAuthKind = autoscaling.GroupName + "/trigger-prometheus-auth-kind" - KedaAutoscaleAnotationPrometheusAuthModes = autoscaling.GroupName + "/trigger-prometheus-auth-modes" + KedaAutoscaleAnnotationPrometheusAddress = autoscaling.GroupName + "/prometheus-address" + KedaAutoscaleAnnotationPrometheusQuery = autoscaling.GroupName + "/prometheus-query" + KedaAutoscaleAnnotationPrometheusAuthName = autoscaling.GroupName + "/trigger-prometheus-auth-name" + KedaAutoscaleAnnotationPrometheusAuthKind = autoscaling.GroupName + "/trigger-prometheus-auth-kind" + KedaAutoscaleAnnotationPrometheusAuthModes = autoscaling.GroupName + "/trigger-prometheus-auth-modes" ) // DesiredScaledObject creates an ScaledObject KEDA resource from a PA resource. @@ -105,12 +105,12 @@ func DesiredScaledObject(pa *autoscalingv1alpha1.PodAutoscaler, config *autoscal if target, ok := pa.Target(); ok { targetQuantity := resource.NewQuantity(int64(target), resource.DecimalSI) var query, address string - if v, ok := pa.Annotations[KedaAutoscaleAnotationPrometheusQuery]; ok { + if v, ok := pa.Annotations[KedaAutoscaleAnnotationPrometheusQuery]; ok { query = v } else { query = fmt.Sprintf("sum(rate(%s{}[1m]))", pa.Metric()) } - if v, ok := pa.Annotations[KedaAutoscaleAnotationPrometheusAddress]; ok { + if v, ok := pa.Annotations[KedaAutoscaleAnnotationPrometheusAddress]; ok { if err := helpers.ParseServerAddress(v); err != nil { return nil, fmt.Errorf("invalid prometheus address: %w", err) } @@ -157,12 +157,12 @@ func getDefaultPrometheusTrigger(annotations map[string]string, address string, var ref *v1alpha1.AuthenticationRef - if v, ok := annotations[KedaAutoscaleAnotationPrometheusAuthName]; ok { + if v, ok := annotations[KedaAutoscaleAnnotationPrometheusAuthName]; ok { ref = &v1alpha1.AuthenticationRef{} ref.Name = v } - if v, ok := annotations[KedaAutoscaleAnotationPrometheusAuthKind]; ok { + if v, ok := annotations[KedaAutoscaleAnnotationPrometheusAuthKind]; ok { if ref == nil { return nil, fmt.Errorf("you need to specify the name as well for authentication") } @@ -172,7 +172,7 @@ func getDefaultPrometheusTrigger(annotations map[string]string, address string, ref.Kind = v } - if v, ok := annotations[KedaAutoscaleAnotationPrometheusAuthModes]; ok { + if v, ok := annotations[KedaAutoscaleAnnotationPrometheusAuthModes]; ok { if ref == nil { return nil, fmt.Errorf("you need to specify the name as well for authentication") } diff --git a/pkg/reconciler/autoscaling/hpa/resources/keda_test.go b/pkg/reconciler/autoscaling/hpa/resources/keda_test.go index fc84d048..42f1a5e1 100644 --- a/pkg/reconciler/autoscaling/hpa/resources/keda_test.go +++ b/pkg/reconciler/autoscaling/hpa/resources/keda_test.go @@ -106,20 +106,20 @@ func TestDesiredScaledObject(t *testing.T) { }, { name: "custom metric with default cm values", paAnnotations: map[string]string{ - autoscaling.MinScaleAnnotationKey: "1", - autoscaling.MaxScaleAnnotationKey: "10", - autoscaling.MetricAnnotationKey: "http_requests_total", - KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", - autoscaling.TargetAnnotationKey: "5", + autoscaling.MinScaleAnnotationKey: "1", + autoscaling.MaxScaleAnnotationKey: "10", + autoscaling.MetricAnnotationKey: "http_requests_total", + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + autoscaling.TargetAnnotationKey: "5", }, wantScaledObject: ScaledObject(helpers.TestNamespace, helpers.TestRevision, WithAnnotations(map[string]string{ - autoscaling.MinScaleAnnotationKey: "1", - autoscaling.MaxScaleAnnotationKey: "10", - autoscaling.MetricAnnotationKey: "http_requests_total", - KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", - autoscaling.TargetAnnotationKey: "5", - autoscaling.ClassAnnotationKey: autoscaling.HPA, + autoscaling.MinScaleAnnotationKey: "1", + autoscaling.MaxScaleAnnotationKey: "10", + autoscaling.MetricAnnotationKey: "http_requests_total", + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + autoscaling.TargetAnnotationKey: "5", + autoscaling.ClassAnnotationKey: autoscaling.HPA, }), WithMaxScale(10), WithMinScale(1), WithPrometheusTrigger(map[string]string{ "namespace": helpers.TestNamespace, "query": "sum(rate(http_requests_total{}[1m]))", @@ -128,63 +128,63 @@ func TestDesiredScaledObject(t *testing.T) { }), WithScaleTargetRef(helpers.TestRevision+"-deployment"), WithHorizontalPodAutoscalerConfig(helpers.TestRevision)), }, { name: "custom metric with bad prometheus address", - paAnnotations: map[string]string{ - autoscaling.MinScaleAnnotationKey: "1", - autoscaling.MaxScaleAnnotationKey: "10", - autoscaling.MetricAnnotationKey: "http_requests_total", - KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", - KedaAutoscaleAnotationPrometheusAddress: "http//9090", - autoscaling.TargetAnnotationKey: "5", - }, - wantErr: true, - }, { - name: "custom metric with bad auth kind", paAnnotations: map[string]string{ autoscaling.MinScaleAnnotationKey: "1", autoscaling.MaxScaleAnnotationKey: "10", autoscaling.MetricAnnotationKey: "http_requests_total", - KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", - KedaAutoscaleAnotationPrometheusAuthKind: "TriggerAuth", - KedaAutoscaleAnotationPrometheusAuthName: "keda-trigger-auth-prometheus", + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + KedaAutoscaleAnnotationPrometheusAddress: "http//9090", autoscaling.TargetAnnotationKey: "5", }, wantErr: true, }, { - name: "custom metric with no auth name", + name: "custom metric with bad auth kind", paAnnotations: map[string]string{ - autoscaling.MinScaleAnnotationKey: "1", - autoscaling.MaxScaleAnnotationKey: "10", - autoscaling.MetricAnnotationKey: "http_requests_total", - KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", - KedaAutoscaleAnotationPrometheusAuthKind: "TriggerAuthentication", - autoscaling.TargetAnnotationKey: "5", + autoscaling.MinScaleAnnotationKey: "1", + autoscaling.MaxScaleAnnotationKey: "10", + autoscaling.MetricAnnotationKey: "http_requests_total", + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + KedaAutoscaleAnnotationPrometheusAuthKind: "TriggerAuth", + KedaAutoscaleAnnotationPrometheusAuthName: "keda-trigger-auth-prometheus", + autoscaling.TargetAnnotationKey: "5", }, wantErr: true, }, { - name: "custom metric with default cm values with authentication", + name: "custom metric with no auth name", paAnnotations: map[string]string{ autoscaling.MinScaleAnnotationKey: "1", autoscaling.MaxScaleAnnotationKey: "10", autoscaling.MetricAnnotationKey: "http_requests_total", - KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + KedaAutoscaleAnnotationPrometheusAuthKind: "TriggerAuthentication", autoscaling.TargetAnnotationKey: "5", - KedaAutoscaleAnotationPrometheusAddress: "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092", - KedaAutoscaleAnotationPrometheusAuthName: "keda-trigger-auth-prometheus", - KedaAutoscaleAnotationPrometheusAuthKind: "TriggerAuthentication", - KedaAutoscaleAnotationPrometheusAuthModes: "bearer", + }, + wantErr: true, + }, { + name: "custom metric with default cm values with authentication", + paAnnotations: map[string]string{ + autoscaling.MinScaleAnnotationKey: "1", + autoscaling.MaxScaleAnnotationKey: "10", + autoscaling.MetricAnnotationKey: "http_requests_total", + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + autoscaling.TargetAnnotationKey: "5", + KedaAutoscaleAnnotationPrometheusAddress: "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092", + KedaAutoscaleAnnotationPrometheusAuthName: "keda-trigger-auth-prometheus", + KedaAutoscaleAnnotationPrometheusAuthKind: "TriggerAuthentication", + KedaAutoscaleAnnotationPrometheusAuthModes: "bearer", }, wantScaledObject: ScaledObject(helpers.TestNamespace, helpers.TestRevision, WithAnnotations(map[string]string{ - autoscaling.MinScaleAnnotationKey: "1", - autoscaling.MaxScaleAnnotationKey: "10", - autoscaling.MetricAnnotationKey: "http_requests_total", - KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", - autoscaling.TargetAnnotationKey: "5", - autoscaling.ClassAnnotationKey: autoscaling.HPA, - KedaAutoscaleAnotationPrometheusAddress: "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092", - KedaAutoscaleAnotationPrometheusAuthName: "keda-trigger-auth-prometheus", - KedaAutoscaleAnotationPrometheusAuthKind: "TriggerAuthentication", - KedaAutoscaleAnotationPrometheusAuthModes: "bearer", + autoscaling.MinScaleAnnotationKey: "1", + autoscaling.MaxScaleAnnotationKey: "10", + autoscaling.MetricAnnotationKey: "http_requests_total", + KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + autoscaling.TargetAnnotationKey: "5", + autoscaling.ClassAnnotationKey: autoscaling.HPA, + KedaAutoscaleAnnotationPrometheusAddress: "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092", + KedaAutoscaleAnnotationPrometheusAuthName: "keda-trigger-auth-prometheus", + KedaAutoscaleAnnotationPrometheusAuthKind: "TriggerAuthentication", + KedaAutoscaleAnnotationPrometheusAuthModes: "bearer", }), WithMaxScale(10), WithMinScale(1), WithScaleTargetRef(helpers.TestRevision+"-deployment"), WithAuthPrometheusTrigger(map[string]string{ "query": "sum(rate(http_requests_total{}[1m]))", diff --git a/test/e2e/autoscale_custom_test.go b/test/e2e/autoscale_custom_test.go index 2ffb2cec..4e5305e7 100644 --- a/test/e2e/autoscale_custom_test.go +++ b/test/e2e/autoscale_custom_test.go @@ -76,13 +76,13 @@ func setupCustomHPASvc(t *testing.T, metric string, target int) *TestContext { []rtesting.ServiceOption{ withConfigLabels(map[string]string{"metrics-test": "metrics-test"}), rtesting.WithConfigAnnotations(map[string]string{ - autoscaling.ClassAnnotationKey: autoscaling.HPA, - autoscaling.MetricAnnotationKey: metric, - autoscaling.TargetAnnotationKey: strconv.Itoa(target), - autoscaling.MinScaleAnnotationKey: "1", - autoscaling.MaxScaleAnnotationKey: fmt.Sprintf("%d", int(maxPods)), - autoscaling.WindowAnnotationKey: "20s", - resources2.KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))", + autoscaling.ClassAnnotationKey: autoscaling.HPA, + autoscaling.MetricAnnotationKey: metric, + autoscaling.TargetAnnotationKey: strconv.Itoa(target), + autoscaling.MinScaleAnnotationKey: "1", + autoscaling.MaxScaleAnnotationKey: fmt.Sprintf("%d", int(maxPods)), + autoscaling.WindowAnnotationKey: "20s", + resources2.KedaAutoscaleAnnotationPrometheusQuery: fmt.Sprintf("sum(rate(http_requests_total{namespace='%s'}[1m]))", test.ServingFlags.TestNamespace), }), rtesting.WithResourceRequirements(corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("30m"),