BYO scaledobject (#31)

* BYO scaledobject

* update docs

* fix spelling etc

DEVELOPMENT.md

@@ -169,10 +169,36 @@ prometheus-prometheus-kube-prometheus-prometheus-0       2/2     Running       0
 prometheus-prometheus-node-exporter-q5qzv                1/1     Running       0          126m
 ```
 
-## Roadmap
+### Bring Your Own ScaledObject
+
+You can bring your own ScaledObject by either disabling globally the auto-creation mode or by setting the `autoscaling.knative.dev/scaled-object-auto-create` annotation to `false` in the service (`spec.template.metadata` field).
+
+Then at any time you can create the ScaledObject with the desired configuration as follows:
+
+```yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  annotations:
+    "autoscaling.knative.dev/class": "hpa.autoscaling.knative.dev"
+  name: metrics-test-00001
+  namespace: test
+spec:
+  advanced:
+    horizontalPodAutoscalerConfig:
+      name: metrics-test-00001
+    scalingModifiers: {}
+  maxReplicaCount: 10
+  minReplicaCount: 1
+  scaleTargetRef:
+    name: metrics-test-00001-deployment
+  triggers:
+    - metadata:
+        namespace: test
+        query: sum(rate(http_requests_total{namespace="test"}[1m]))
+        serverAddress: http://prometheus-operated.default.svc:9090
+        threshold: "5"
+      type: prometheus
+```
 
-- Support more functionality wrt hpa configuration compared to the original autoscaler-hpa
-- Add e2e tests (Kind)
-- Allow user to specify a ScaledObject instead of auto-creating it (non managed mode). Useful also for the KServe integration.
-- HA support
-- OCP Instructions
+The annotation for the scaling class is required so reconciliation is triggered for the HPA that is generated by KEDA.

config/config.yaml

@@ -40,3 +40,11 @@ data:
 
     # configures the default Prometheus address if no is specified per service
     autoscaler.keda.prometheus-address: "http://prometheus-operated.default.svc:9090"
+
+    # configures the globally default mode for creating ScaledObject. Default is true.
+    # If you set this to false make sure that you have a ScaledObject for each service
+    # and existing services with a previously automatically created ScaledObject should
+    # have their Knative service annotation `autoscaling.knative.dev/scaled-object-auto-create` set to true.
+    # By setting `autoscaling.knative.dev/scaled-object-auto-create` at the Knative Service level you can bypass
+    # this configuration and by setting to false you can bring your own scaled object.
+    autoscaler.keda.scaledobject-autocreate: "true"

pkg/reconciler/autoscaling/hpa/config/autoscaler_keda.go

@@ -34,16 +34,19 @@ const (
 // AutoscalerKedaConfig contains autoscaler keda related configuration defined in the
 // `config-autoscaler-keda` config map.
 type AutoscalerKedaConfig struct {
-	PrometheusAddress string
+	PrometheusAddress        string
+	ShouldCreateScaledObject bool
 }
 
 // NewAutoscalerKedaConfigFromConfigMap creates an AutoscalerKedaConfig from the supplied ConfigMap
 func NewConfigFromMap(data map[string]string) (*AutoscalerKedaConfig, error) {
 	config := &AutoscalerKedaConfig{
-		PrometheusAddress: DefaultPrometheusAddress,
+		PrometheusAddress:        DefaultPrometheusAddress,
+		ShouldCreateScaledObject: true,
 	}
 	if err := cm.Parse(data,
 		cm.AsString("autoscaler.keda.prometheus-address", &config.PrometheusAddress),
+		cm.AsBool("autoscaler.keda.scaledobject-autocreate", &config.ShouldCreateScaledObject),
 	); err != nil {
 		return nil, fmt.Errorf("failed to parse data: %w", err)
 	}

pkg/reconciler/autoscaling/hpa/keda_hpa.go

@@ -19,6 +19,10 @@ package hpa
 import (
 	"context"
 	"fmt"
+	"strconv"
+
+	"github.com/kedacore/keda/v2/apis/keda/v1alpha1"
+	"knative.dev/serving/pkg/apis/autoscaling"
 
 	v2 "k8s.io/api/autoscaling/v2"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -42,7 +46,10 @@ import (
 	"knative.dev/autoscaler-keda/pkg/reconciler/autoscaling/hpa/resources"
 )
 
-const allActivators = 0
+const (
+	allActivators                     = 0
+	KedaAutoscaleAnnotationAutocreate = autoscaling.GroupName + "/scaled-object-auto-create"
+)
 
 // Reconciler implements the control loop for the HPA resources.
 type Reconciler struct {
@@ -64,41 +71,58 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, pa *autoscalingv1alpha1.
 
 	logger := logging.FromContext(ctx)
 
+	var scaledObj *v1alpha1.ScaledObject
 	var hpa *v2.HorizontalPodAutoscaler
+	shouldCreateScaledObject := true
 
-	dScaledObject, err := resources.DesiredScaledObject(pa, hpaconfig.FromContext(ctx).Autoscaler, hpaconfig.FromContext(ctx).AutoscalerKeda)
-	if err != nil {
-		return fmt.Errorf("failed to contruct desiredScaledObject: %w", err)
+	if hpaconfig.FromContext(ctx).AutoscalerKeda != nil {
+		shouldCreateScaledObject = hpaconfig.FromContext(ctx).AutoscalerKeda.ShouldCreateScaledObject
 	}
 
-	scaledObj, err := c.kedaLister.ScaledObjects(pa.Namespace).Get(dScaledObject.Name)
-	if errors.IsNotFound(err) {
-		logger.Infof("Creating Scaled Object %q", dScaledObject.Name)
-		if scaledObj, err = c.kedaClient.KedaV1alpha1().ScaledObjects(dScaledObject.Namespace).Create(ctx, dScaledObject, metav1.CreateOptions{}); err != nil {
-			pa.Status.MarkResourceFailedCreation("ScaledObject", dScaledObject.Name)
-			return fmt.Errorf("failed to create ScaledObject: %w", err)
+	if v, ok := pa.Annotations[KedaAutoscaleAnnotationAutocreate]; ok {
+		if b, err := strconv.ParseBool(v); err == nil {
+			shouldCreateScaledObject = b
+		} else {
+			logger.Warnf("Failed to parse annotation %q value %q as boolean: %v", KedaAutoscaleAnnotationAutocreate, v, err)
 		}
-	} else if err != nil {
-		return fmt.Errorf("failed to get ScaledObject: %w", err)
-	} else if !metav1.IsControlledBy(scaledObj, pa) {
-		// Surface an error in the PodAutoscaler's status, and return an error.
-		pa.Status.MarkResourceNotOwned("ScaledObject", dScaledObject.Name)
-		return fmt.Errorf("PodAutoscaler: %q does not own ScaledObject: %q", pa.Name, dScaledObject.Name)
 	}
-	if !equality.Semantic.DeepEqual(dScaledObject.Spec, scaledObj.Spec) {
-		logger.Infof("Updating ScaledObject %q", dScaledObject.Name)
-		update := scaledObj.DeepCopy()
-		update.Spec = dScaledObject.Spec
-		if _, err := c.kedaClient.KedaV1alpha1().ScaledObjects(pa.Namespace).Update(ctx, update, metav1.UpdateOptions{}); err != nil {
-			return fmt.Errorf("failed to update ScaledObject: %w", err)
+
+	if shouldCreateScaledObject {
+		dScaledObject, err := resources.DesiredScaledObject(pa, hpaconfig.FromContext(ctx).Autoscaler, hpaconfig.FromContext(ctx).AutoscalerKeda)
+		if err != nil {
+			return fmt.Errorf("failed to contruct desiredScaledObject: %w", err)
+		}
+
+		scaledObj, err = c.kedaLister.ScaledObjects(pa.Namespace).Get(dScaledObject.Name)
+		if errors.IsNotFound(err) {
+			logger.Infof("Creating Scaled Object %q", dScaledObject.Name)
+			if scaledObj, err = c.kedaClient.KedaV1alpha1().ScaledObjects(dScaledObject.Namespace).Create(ctx, dScaledObject, metav1.CreateOptions{}); err != nil {
+				pa.Status.MarkResourceFailedCreation("ScaledObject", dScaledObject.Name)
+				return fmt.Errorf("failed to create ScaledObject: %w", err)
+			}
+		} else if err != nil {
+			return fmt.Errorf("failed to get ScaledObject: %w", err)
+		} else if !metav1.IsControlledBy(scaledObj, pa) {
+			// Surface an error in the PodAutoscaler's status, and return an error.
+			pa.Status.MarkResourceNotOwned("ScaledObject", dScaledObject.Name)
+			return fmt.Errorf("PodAutoscaler: %q does not own ScaledObject: %q", pa.Name, dScaledObject.Name)
+		}
+		if !equality.Semantic.DeepEqual(dScaledObject.Spec, scaledObj.Spec) {
+			logger.Infof("Updating ScaledObject %q", dScaledObject.Name)
+			update := scaledObj.DeepCopy()
+			update.Spec = dScaledObject.Spec
+			if _, err := c.kedaClient.KedaV1alpha1().ScaledObjects(pa.Namespace).Update(ctx, update, metav1.UpdateOptions{}); err != nil {
+				return fmt.Errorf("failed to update ScaledObject: %w", err)
+			}
 		}
 	}
-	hpa, err = c.hpaLister.HorizontalPodAutoscalers(pa.Namespace).Get(pa.Name)
+	hpa, err := c.hpaLister.HorizontalPodAutoscalers(pa.Namespace).Get(pa.Name)
 	if errors.IsNotFound(err) {
+		logger.Infof("Skipping HPA %q", pa.Name)
 		return nil // skip wait to be triggered by hpa events eg. creation
 	}
 
-	if scaledObj.Spec.MinReplicaCount != nil {
+	if scaledObj != nil && scaledObj.Spec.MinReplicaCount != nil {
 		if hpa.Status.DesiredReplicas < *scaledObj.Spec.MinReplicaCount {
 			return nil // skip wait to be triggered by hpa events
 		}

pkg/reconciler/autoscaling/hpa/resources/keda.go

@@ -37,11 +37,11 @@ import (
 )
 
 const (
-	KedaAutoscaleAnotationPrometheusAddress   = autoscaling.GroupName + "/prometheus-address"
-	KedaAutoscaleAnotationPrometheusQuery     = autoscaling.GroupName + "/prometheus-query"
-	KedaAutoscaleAnotationPrometheusAuthName  = autoscaling.GroupName + "/trigger-prometheus-auth-name"
-	KedaAutoscaleAnotationPrometheusAuthKind  = autoscaling.GroupName + "/trigger-prometheus-auth-kind"
-	KedaAutoscaleAnotationPrometheusAuthModes = autoscaling.GroupName + "/trigger-prometheus-auth-modes"
+	KedaAutoscaleAnnotationPrometheusAddress   = autoscaling.GroupName + "/prometheus-address"
+	KedaAutoscaleAnnotationPrometheusQuery     = autoscaling.GroupName + "/prometheus-query"
+	KedaAutoscaleAnnotationPrometheusAuthName  = autoscaling.GroupName + "/trigger-prometheus-auth-name"
+	KedaAutoscaleAnnotationPrometheusAuthKind  = autoscaling.GroupName + "/trigger-prometheus-auth-kind"
+	KedaAutoscaleAnnotationPrometheusAuthModes = autoscaling.GroupName + "/trigger-prometheus-auth-modes"
 )
 
 // DesiredScaledObject creates an ScaledObject KEDA resource from a PA resource.
@@ -105,12 +105,12 @@ func DesiredScaledObject(pa *autoscalingv1alpha1.PodAutoscaler, config *autoscal
 			if target, ok := pa.Target(); ok {
 				targetQuantity := resource.NewQuantity(int64(target), resource.DecimalSI)
 				var query, address string
-				if v, ok := pa.Annotations[KedaAutoscaleAnotationPrometheusQuery]; ok {
+				if v, ok := pa.Annotations[KedaAutoscaleAnnotationPrometheusQuery]; ok {
 					query = v
 				} else {
 					query = fmt.Sprintf("sum(rate(%s{}[1m]))", pa.Metric())
 				}
-				if v, ok := pa.Annotations[KedaAutoscaleAnotationPrometheusAddress]; ok {
+				if v, ok := pa.Annotations[KedaAutoscaleAnnotationPrometheusAddress]; ok {
 					if err := helpers.ParseServerAddress(v); err != nil {
 						return nil, fmt.Errorf("invalid prometheus address: %w", err)
 					}
@@ -157,12 +157,12 @@ func getDefaultPrometheusTrigger(annotations map[string]string, address string,
 
 	var ref *v1alpha1.AuthenticationRef
 
-	if v, ok := annotations[KedaAutoscaleAnotationPrometheusAuthName]; ok {
+	if v, ok := annotations[KedaAutoscaleAnnotationPrometheusAuthName]; ok {
 		ref = &v1alpha1.AuthenticationRef{}
 		ref.Name = v
 	}
 
-	if v, ok := annotations[KedaAutoscaleAnotationPrometheusAuthKind]; ok {
+	if v, ok := annotations[KedaAutoscaleAnnotationPrometheusAuthKind]; ok {
 		if ref == nil {
 			return nil, fmt.Errorf("you need to specify the name as well for authentication")
 		}
@@ -172,7 +172,7 @@ func getDefaultPrometheusTrigger(annotations map[string]string, address string,
 		ref.Kind = v
 	}
 
-	if v, ok := annotations[KedaAutoscaleAnotationPrometheusAuthModes]; ok {
+	if v, ok := annotations[KedaAutoscaleAnnotationPrometheusAuthModes]; ok {
 		if ref == nil {
 			return nil, fmt.Errorf("you need to specify the name as well for authentication")
 		}

pkg/reconciler/autoscaling/hpa/resources/keda_test.go

@@ -106,20 +106,20 @@ func TestDesiredScaledObject(t *testing.T) {
 	}, {
 		name: "custom metric with default cm values",
 		paAnnotations: map[string]string{
-			autoscaling.MinScaleAnnotationKey:     "1",
-			autoscaling.MaxScaleAnnotationKey:     "10",
-			autoscaling.MetricAnnotationKey:       "http_requests_total",
-			KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))",
-			autoscaling.TargetAnnotationKey:       "5",
+			autoscaling.MinScaleAnnotationKey:      "1",
+			autoscaling.MaxScaleAnnotationKey:      "10",
+			autoscaling.MetricAnnotationKey:        "http_requests_total",
+			KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))",
+			autoscaling.TargetAnnotationKey:        "5",
 		},
 		wantScaledObject: ScaledObject(helpers.TestNamespace,
 			helpers.TestRevision, WithAnnotations(map[string]string{
-				autoscaling.MinScaleAnnotationKey:     "1",
-				autoscaling.MaxScaleAnnotationKey:     "10",
-				autoscaling.MetricAnnotationKey:       "http_requests_total",
-				KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))",
-				autoscaling.TargetAnnotationKey:       "5",
-				autoscaling.ClassAnnotationKey:        autoscaling.HPA,
+				autoscaling.MinScaleAnnotationKey:      "1",
+				autoscaling.MaxScaleAnnotationKey:      "10",
+				autoscaling.MetricAnnotationKey:        "http_requests_total",
+				KedaAutoscaleAnnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))",
+				autoscaling.TargetAnnotationKey:        "5",
+				autoscaling.ClassAnnotationKey:         autoscaling.HPA,
 			}), WithMaxScale(10), WithMinScale(1), WithPrometheusTrigger(map[string]string{
 				"namespace":     helpers.TestNamespace,
 				"query":         "sum(rate(http_requests_total{}[1m]))",
@@ -128,63 +128,63 @@ func TestDesiredScaledObject(t *testing.T) {
 			}), WithScaleTargetRef(helpers.TestRevision+"-deployment"), WithHorizontalPodAutoscalerConfig(helpers.TestRevision)),
 	}, {
 		name: "custom metric with bad prometheus address",
-		paAnnotations: map[string]string{
-			autoscaling.MinScaleAnnotationKey:       "1",
-			autoscaling.MaxScaleAnnotationKey:       "10",
-			autoscaling.MetricAnnotationKey:         "http_requests_total",
-			KedaAutoscaleAnotationPrometheusQuery:   "sum(rate(http_requests_total{}[1m]))",
-			KedaAutoscaleAnotationPrometheusAddress: "http//9090",
-			autoscaling.TargetAnnotationKey:         "5",
-		},
-		wantErr: true,
-	}, {
-		name: "custom metric with bad auth kind",
 		paAnnotations: map[string]string{
 			autoscaling.MinScaleAnnotationKey:        "1",
 			autoscaling.MaxScaleAnnotationKey:        "10",
 			autoscaling.MetricAnnotationKey:          "http_requests_total",
-			KedaAutoscaleAnotationPrometheusQuery:    "sum(rate(http_requests_total{}[1m]))",
-			KedaAutoscaleAnotationPrometheusAuthKind: "TriggerAuth",
-			KedaAutoscaleAnotationPrometheusAuthName: "keda-trigger-auth-prometheus",
+			KedaAutoscaleAnnotationPrometheusQuery:   "sum(rate(http_requests_total{}[1m]))",
+			KedaAutoscaleAnnotationPrometheusAddress: "http//9090",
 			autoscaling.TargetAnnotationKey:          "5",
 		},
 		wantErr: true,
 	}, {
-		name: "custom metric with no auth name",
+		name: "custom metric with bad auth kind",
 		paAnnotations: map[string]string{
-			autoscaling.MinScaleAnnotationKey:        "1",
-			autoscaling.MaxScaleAnnotationKey:        "10",
-			autoscaling.MetricAnnotationKey:          "http_requests_total",
-			KedaAutoscaleAnotationPrometheusQuery:    "sum(rate(http_requests_total{}[1m]))",
-			KedaAutoscaleAnotationPrometheusAuthKind: "TriggerAuthentication",
-			autoscaling.TargetAnnotationKey:          "5",
+			autoscaling.MinScaleAnnotationKey:         "1",
+			autoscaling.MaxScaleAnnotationKey:         "10",
+			autoscaling.MetricAnnotationKey:           "http_requests_total",
+			KedaAutoscaleAnnotationPrometheusQuery:    "sum(rate(http_requests_total{}[1m]))",
+			KedaAutoscaleAnnotationPrometheusAuthKind: "TriggerAuth",
+			KedaAutoscaleAnnotationPrometheusAuthName: "keda-trigger-auth-prometheus",
+			autoscaling.TargetAnnotationKey:           "5",
 		},
 		wantErr: true,
 	}, {
-		name: "custom metric with default cm values with authentication",
+		name: "custom metric with no auth name",
 		paAnnotations: map[string]string{
 			autoscaling.MinScaleAnnotationKey:         "1",
 			autoscaling.MaxScaleAnnotationKey:         "10",
 			autoscaling.MetricAnnotationKey:           "http_requests_total",
-			KedaAutoscaleAnotationPrometheusQuery:     "sum(rate(http_requests_total{}[1m]))",
+			KedaAutoscaleAnnotationPrometheusQuery:    "sum(rate(http_requests_total{}[1m]))",
+			KedaAutoscaleAnnotationPrometheusAuthKind: "TriggerAuthentication",
 			autoscaling.TargetAnnotationKey:           "5",
-			KedaAutoscaleAnotationPrometheusAddress:   "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092",
-			KedaAutoscaleAnotationPrometheusAuthName:  "keda-trigger-auth-prometheus",
-			KedaAutoscaleAnotationPrometheusAuthKind:  "TriggerAuthentication",
-			KedaAutoscaleAnotationPrometheusAuthModes: "bearer",
+		},
+		wantErr: true,
+	}, {
+		name: "custom metric with default cm values with authentication",
+		paAnnotations: map[string]string{
+			autoscaling.MinScaleAnnotationKey:          "1",
+			autoscaling.MaxScaleAnnotationKey:          "10",
+			autoscaling.MetricAnnotationKey:            "http_requests_total",
+			KedaAutoscaleAnnotationPrometheusQuery:     "sum(rate(http_requests_total{}[1m]))",
+			autoscaling.TargetAnnotationKey:            "5",
+			KedaAutoscaleAnnotationPrometheusAddress:   "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092",
+			KedaAutoscaleAnnotationPrometheusAuthName:  "keda-trigger-auth-prometheus",
+			KedaAutoscaleAnnotationPrometheusAuthKind:  "TriggerAuthentication",
+			KedaAutoscaleAnnotationPrometheusAuthModes: "bearer",
 		},
 		wantScaledObject: ScaledObject(helpers.TestNamespace,
 			helpers.TestRevision, WithAnnotations(map[string]string{
-				autoscaling.MinScaleAnnotationKey:         "1",
-				autoscaling.MaxScaleAnnotationKey:         "10",
-				autoscaling.MetricAnnotationKey:           "http_requests_total",
-				KedaAutoscaleAnotationPrometheusQuery:     "sum(rate(http_requests_total{}[1m]))",
-				autoscaling.TargetAnnotationKey:           "5",
-				autoscaling.ClassAnnotationKey:            autoscaling.HPA,
-				KedaAutoscaleAnotationPrometheusAddress:   "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092",
-				KedaAutoscaleAnotationPrometheusAuthName:  "keda-trigger-auth-prometheus",
-				KedaAutoscaleAnotationPrometheusAuthKind:  "TriggerAuthentication",
-				KedaAutoscaleAnotationPrometheusAuthModes: "bearer",
+				autoscaling.MinScaleAnnotationKey:          "1",
+				autoscaling.MaxScaleAnnotationKey:          "10",
+				autoscaling.MetricAnnotationKey:            "http_requests_total",
+				KedaAutoscaleAnnotationPrometheusQuery:     "sum(rate(http_requests_total{}[1m]))",
+				autoscaling.TargetAnnotationKey:            "5",
+				autoscaling.ClassAnnotationKey:             autoscaling.HPA,
+				KedaAutoscaleAnnotationPrometheusAddress:   "https://thanos-querier.openshift-monitoring.svc.cluster.local:9092",
+				KedaAutoscaleAnnotationPrometheusAuthName:  "keda-trigger-auth-prometheus",
+				KedaAutoscaleAnnotationPrometheusAuthKind:  "TriggerAuthentication",
+				KedaAutoscaleAnnotationPrometheusAuthModes: "bearer",
 			}), WithMaxScale(10), WithMinScale(1), WithScaleTargetRef(helpers.TestRevision+"-deployment"),
 			WithAuthPrometheusTrigger(map[string]string{
 				"query":         "sum(rate(http_requests_total{}[1m]))",

test/e2e/autoscale_custom_test.go

@@ -76,13 +76,13 @@ func setupCustomHPASvc(t *testing.T, metric string, target int) *TestContext {
 		[]rtesting.ServiceOption{
 			withConfigLabels(map[string]string{"metrics-test": "metrics-test"}),
 			rtesting.WithConfigAnnotations(map[string]string{
-				autoscaling.ClassAnnotationKey:                   autoscaling.HPA,
-				autoscaling.MetricAnnotationKey:                  metric,
-				autoscaling.TargetAnnotationKey:                  strconv.Itoa(target),
-				autoscaling.MinScaleAnnotationKey:                "1",
-				autoscaling.MaxScaleAnnotationKey:                fmt.Sprintf("%d", int(maxPods)),
-				autoscaling.WindowAnnotationKey:                  "20s",
-				resources2.KedaAutoscaleAnotationPrometheusQuery: "sum(rate(http_requests_total{}[1m]))",
+				autoscaling.ClassAnnotationKey:                    autoscaling.HPA,
+				autoscaling.MetricAnnotationKey:                   metric,
+				autoscaling.TargetAnnotationKey:                   strconv.Itoa(target),
+				autoscaling.MinScaleAnnotationKey:                 "1",
+				autoscaling.MaxScaleAnnotationKey:                 fmt.Sprintf("%d", int(maxPods)),
+				autoscaling.WindowAnnotationKey:                   "20s",
+				resources2.KedaAutoscaleAnnotationPrometheusQuery: fmt.Sprintf("sum(rate(http_requests_total{namespace='%s'}[1m]))", test.ServingFlags.TestNamespace),
 			}), rtesting.WithResourceRequirements(corev1.ResourceRequirements{
 				Requests: corev1.ResourceList{
 					corev1.ResourceCPU:    resource.MustParse("30m"),