add replicas.idle to set idleReplicaCount on SO

Signed-off-by: Luca Kuendig <luca.kuendig@bedag.ch>
kedacore · Mar 9, 2023 · 55f4b8c · 55f4b8c
1 parent 2b4accc
commit 55f4b8c
Show file tree

Hide file tree

Showing 10 changed files with 144 additions and 75 deletions.
diff --git a/config/crd/bases/http.keda.sh_httpscaledobjects.yaml b/config/crd/bases/http.keda.sh_httpscaledobjects.yaml
@@ -33,6 +33,9 @@ spec:
     - jsonPath: .spec.replicas.max
       name: MaxReplicas
       type: integer
+    - jsonPath: .spec.replicas.idle
+      name: IdleReplicas
+      type: integer
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
@@ -66,6 +69,11 @@ spec:
               replicas:
                 description: (optional) Replica information
                 properties:
+                  idle:
+                    description: Amount of replicas to have in the deployment while
+                      being idle
+                    format: int32
+                    type: integer
                   max:
                     description: Maximum amount of replicas to have in the deployment
                       (Default 100)

diff --git a/e2e/k8s.go b/e2e/k8s.go
@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/codeskyblue/go-sh"
-	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
 	"github.com/pkg/errors"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -33,7 +32,28 @@ func deleteNS(ns string) error {
 	return sh.Command("kubectl", "delete", "namespace", ns).Run()
 }
 
-func getScaledObject(ctx context.Context, cl client.Client, ns string, name string) error {
-	var scaledObject kedav1alpha1.ScaledObject
-	return cl.Get(ctx, k8s.ObjKey(ns, name), &scaledObject)
+func getScaledObject(
+	ctx context.Context,
+	cl client.Client,
+	ns,
+	name string,
+) error {
+	scaledObject, err := k8s.NewScaledObject(
+		ns,
+		name,
+		"",
+		"",
+		"",
+		1,
+		2,
+		0,
+		30,
+	)
+	if err != nil {
+		return err
+	}
+	if err := cl.Get(ctx, k8s.ObjKey(ns, name), scaledObject); err != nil {
+		return err
+	}
+	return nil
 }
diff --git a/examples/xkcd/templates/httpscaledobject.yaml b/examples/xkcd/templates/httpscaledobject.yaml
@@ -12,3 +12,4 @@ spec:
     replicas:
         min: {{ .Values.autoscaling.http.minReplicas }}
         max: {{ .Values.autoscaling.http.maxReplicas }}
+        idle: {{ .Values.autoscaling.http.idleReplicas }}
diff --git a/examples/xkcd/values.yaml b/examples/xkcd/values.yaml
@@ -46,3 +46,4 @@ autoscaling:
   http:
     minReplicas: 0
     maxReplicas: 10
+    idleReplicas: 0
diff --git a/operator/api/v1alpha1/httpscaledobject_types.go b/operator/api/v1alpha1/httpscaledobject_types.go
@@ -83,7 +83,9 @@ type ReplicaStruct struct {
 	// Minimum amount of replicas to have in the deployment (Default 0)
 	Min *int32 `json:"min,omitempty" description:"Minimum amount of replicas to have in the deployment (Default 0)"`
 	// Maximum amount of replicas to have in the deployment (Default 100)
-	Max *int32 `json:"max,omitempty" description:"Maximum amount of replicas to have in the deployment (Default 100)"`
+	Max int32 `json:"max,omitempty" description:"Maximum amount of replicas to have in the deployment (Default 100)"`
+	// Amount of replicas to have in the deployment while being idle
+	Idle int32 `json:"idle,omitempty" description:"Amount of replicas to have in the deployment while being idle"`
 }
 
 // HTTPScaledObjectSpec defines the desired state of HTTPScaledObject
@@ -133,6 +135,7 @@ type HTTPScaledObjectStatus struct {
 // +kubebuilder:printcolumn:name="ScaleTargetPort",type="integer",JSONPath=".spec.scaleTargetRef"
 // +kubebuilder:printcolumn:name="MinReplicas",type="integer",JSONPath=".spec.replicas.min"
 // +kubebuilder:printcolumn:name="MaxReplicas",type="integer",JSONPath=".spec.replicas.max"
+// +kubebuilder:printcolumn:name="IdleReplicas",type="integer",JSONPath=".spec.replicas.idle"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 // +kubebuilder:printcolumn:name="Active",type="string",JSONPath=".status.conditions[?(@.type==\"HTTPScaledObjectIsReady\")].status"
 

diff --git a/operator/controllers/scaled_object.go b/operator/controllers/scaled_object.go
@@ -26,22 +26,16 @@ func createOrUpdateScaledObject(
 	httpso *v1alpha1.HTTPScaledObject,
 ) error {
 	logger.Info("Creating scaled objects", "external scaler host name", externalScalerHostName)
-
-	var minReplicaCount *int32
-	var maxReplicaCount *int32
-	if replicas := httpso.Spec.Replicas; replicas != nil {
-		minReplicaCount = replicas.Min
-		maxReplicaCount = replicas.Max
-	}
-
-	appScaledObject := k8s.NewScaledObject(
+	logger.Info("This is the current httpso object", "output", httpso.Spec)
+	appScaledObject, appErr := k8s.NewScaledObject(
 		httpso.GetNamespace(),
 		fmt.Sprintf("%s-app", httpso.GetName()), // HTTPScaledObject name is the same as the ScaledObject name
 		httpso.Spec.ScaleTargetRef.Deployment,
 		externalScalerHostName,
 		httpso.Spec.Host,
-		minReplicaCount,
-		maxReplicaCount,
+		httpso.Spec.Replicas.Min,
+		httpso.Spec.Replicas.Max,
+		httpso.Spec.Replicas.Idle,
 		httpso.Spec.CooldownPeriod,
 	)
 

diff --git a/operator/controllers/scaled_object_test.go b/operator/controllers/scaled_object_test.go
@@ -56,32 +56,29 @@ func TestCreateOrUpdateScaledObject(t *testing.T) {
 		metadata.Name,
 	)
 
-	var minReplicaCount *int32
-	var maxReplicaCount *int32
-	if replicas := testInfra.httpso.Spec.Replicas; replicas != nil {
-		minReplicaCount = replicas.Min
-		maxReplicaCount = replicas.Max
-	}
-
-	r.EqualValues(
-		minReplicaCount,
-		spec.MinReplicaCount,
+	// HTTPScaledObject min/max/idle replicas are int32s,
+	// but the ScaledObject's spec is decoded into
+	// an *unsructured.Unstructured (basically a map[string]interface{})
+	// which is an int64. we need to convert the
+	// HTTPScaledObject's values into int64s before we compare
+	r.Equal(
+		int64(testInfra.httpso.Spec.Replicas.Min),
+		spec["minReplicaCount"],
 	)
 	r.EqualValues(
 		maxReplicaCount,
 		spec.MaxReplicaCount,
 	)
+	r.Equal(
+		int64(testInfra.httpso.Spec.Replicas.Idle),
+		spec["idleReplicaCount"],
+	)
 
 	// now update the min and max replicas on the httpso
 	// and call createOrUpdateScaledObject again
-	if spec := &testInfra.httpso.Spec; spec.Replicas == nil {
-		spec.Replicas = &v1alpha1.ReplicaStruct{
-			Min: new(int32),
-			Max: new(int32),
-		}
-	}
-	*testInfra.httpso.Spec.Replicas.Min++
-	*testInfra.httpso.Spec.Replicas.Max++
+	testInfra.httpso.Spec.Replicas.Min++
+	testInfra.httpso.Spec.Replicas.Max++
+	testInfra.httpso.Spec.Replicas.Idle++
 	r.NoError(createOrUpdateScaledObject(
 		testInfra.ctx,
 		testInfra.cl,
@@ -107,6 +104,10 @@ func TestCreateOrUpdateScaledObject(t *testing.T) {
 		*testInfra.httpso.Spec.Replicas.Max,
 		*spec.MaxReplicaCount,
 	)
+	r.Equal(
+		int64(testInfra.httpso.Spec.Replicas.Idle),
+		spec["idleReplicaCount"],
+	)
 }
 
 func getSO(

diff --git a/operator/controllers/suite_test.go b/operator/controllers/suite_test.go
@@ -31,6 +31,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
+	"github.com/kedacore/http-add-on/operator/api/v1alpha1"
 	httpv1alpha1 "github.com/kedacore/http-add-on/operator/api/v1alpha1"
 	// +kubebuilder:scaffold:imports
 )
@@ -85,6 +86,11 @@ func newCommonTestInfra(namespace, appName string) *commonTestInfra {
 				Service:    appName,
 				Port:       8081,
 			},
+			Replicas: v1alpha1.ReplicaStruct{
+				Min:  0,
+				Max:  20,
+				Idle: 0,
+			},
 		},
 	}
 

diff --git a/pkg/k8s/scaledobject.go b/pkg/k8s/scaledobject.go
@@ -1,10 +1,11 @@
 package k8s
 
 import (
-	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
-	appsv1 "k8s.io/api/apps/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/utils/pointer"
+	"bytes"
+	"text/template"
+
+	"gopkg.in/yaml.v2"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
 const (
@@ -22,42 +23,51 @@ func NewScaledObject(
 	deploymentName string,
 	scalerAddress string,
 	host string,
-	minReplicas *int32,
-	maxReplicas *int32,
-	cooldownPeriod *int32,
-) *kedav1alpha1.ScaledObject {
-	return &kedav1alpha1.ScaledObject{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: kedav1alpha1.SchemeGroupVersion.Identifier(),
-			Kind:       ObjectKind(&kedav1alpha1.ScaledObject{}),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: namespace,
-			Name:      name,
-			Labels:    Labels(name),
-		},
-		Spec: kedav1alpha1.ScaledObjectSpec{
-			ScaleTargetRef: &kedav1alpha1.ScaleTarget{
-				APIVersion: appsv1.SchemeGroupVersion.Identifier(),
-				Kind:       ObjectKind(&appsv1.Deployment{}),
-				Name:       deploymentName,
-			},
-			PollingInterval: pointer.Int32(soPollingInterval),
-			CooldownPeriod:  cooldownPeriod,
-			MinReplicaCount: minReplicas,
-			MaxReplicaCount: maxReplicas,
-			Advanced: &kedav1alpha1.AdvancedConfig{
-				RestoreToOriginalReplicaCount: true,
-			},
-			Triggers: []kedav1alpha1.ScaleTriggers{
-				{
-					Type: soTriggerType,
-					Metadata: map[string]string{
-						mkScalerAddress: scalerAddress,
-						mkHost:          host,
-					},
-				},
-			},
-		},
+	minReplicas,
+	maxReplicas int32,
+	idleReplicas int32,
+	cooldownPeriod int32,
+) (*unstructured.Unstructured, error) {
+	// https://keda.sh/docs/1.5/faq/
+	// https://github.com/kedacore/keda/blob/aa0ea79450a1c7549133aab46f5b916efa2364ab/api/v1alpha1/scaledobject_types.go
+	//
+	// unstructured.Unstructured only supports specific types in it. see here for the list:
+	// https://github.com/kubernetes/apimachinery/blob/v0.17.12/pkg/runtime/converter.go#L449-L476
+	typedLabels := Labels(name)
+	labels := map[string]interface{}{}
+	for k, v := range typedLabels {
+		var vIface interface{} = v
+		labels[k] = vIface
+	}
+
+	tpl, err := template.ParseFS(scaledObjectTemplateFS, "templates/scaledobject.yaml")
+	if err != nil {
+		return nil, err
+	}
+
+	var scaledObjectTemplateBuffer bytes.Buffer
+	if tplErr := tpl.Execute(&scaledObjectTemplateBuffer, map[string]interface{}{
+		"Name":           name,
+		"Namespace":      namespace,
+		"Labels":         labels,
+		"MinReplicas":    minReplicas,
+		"MaxReplicas":    maxReplicas,
+		"IdleReplicas":   idleReplicas,
+		"DeploymentName": deploymentName,
+		"ScalerAddress":  scalerAddress,
+		"Host":           host,
+		"CooldownPeriod": cooldownPeriod,
+	}); tplErr != nil {
+		return nil, tplErr
+	}
+
+	var decodedYaml map[string]interface{}
+	decodeErr := yaml.Unmarshal(scaledObjectTemplateBuffer.Bytes(), &decodedYaml)
+	if decodeErr != nil {
+		return nil, decodeErr
 	}
+
+	return &unstructured.Unstructured{
+		Object: decodedYaml,
+	}, nil
 }
diff --git a/pkg/k8s/templates/scaledobject.yaml b/pkg/k8s/templates/scaledobject.yaml
@@ -0,0 +1,25 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: {{ .Name }}
+  namespace: {{ .Namespace }}
+  labels:
+  {{- range $key, $val := .Labels }}
+    {{ $key }}: {{ $val }}
+  {{- end }}
+spec:
+  minReplicaCount: {{ .MinReplicas }}
+  maxReplicaCount: {{ .MaxReplicas }}
+  idleReplicaCount: {{ .IdleReplicas }}
+  cooldownPeriod: {{ .CooldownPeriod }}
+  pollingInterval: 1
+  scaleTargetRef:
+    name: {{ .DeploymentName }}
+    kind: Deployment
+  triggers:
+    - type: external-push
+      metadata:
+        scalerAddress: {{ .ScalerAddress }}
+        host: {{ .Host }}
+  advanced:
+    restoreToOriginalReplicaCount: true