feat: resource and concurrency settings (#2872)

Signed-off-by: Kent Rancourt <kent.rancourt@gmail.com>
akuity · Oct 29, 2024 · f30f66b · f30f66b
1 parent f8179dc
commit f30f66b
Show file tree

Hide file tree

Showing 23 changed files with 304 additions and 88 deletions.
diff --git a/charts/kargo/README.md b/charts/kargo/README.md
diff --git a/charts/kargo/templates/api/deployment.yaml b/charts/kargo/templates/api/deployment.yaml
@@ -59,9 +59,19 @@ spec:
           image: {{ include "kargo.image" . }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           command: ["/usr/local/bin/kargo", "api"]
-          {{- with (concat .Values.global.env .Values.api.env) }}
           env:
-            {{- toYaml . | nindent 10 }}
+          - name: GOMEMLIMIT
+            valueFrom:
+              resourceFieldRef:
+                containerName: api
+                resource: limits.memory
+          - name: GOMAXPROCS
+            valueFrom:
+              resourceFieldRef:
+                containerName: api
+                resource: limits.cpu
+          {{- with (concat .Values.global.env .Values.api.env) }}
+          {{- toYaml . | nindent 10 }}
           {{- end }}
           envFrom:
           - configMapRef:

diff --git a/charts/kargo/templates/controller/configmap.yaml b/charts/kargo/templates/controller/configmap.yaml
@@ -41,4 +41,8 @@ data:
   {{- if .Values.controller.rollouts.integrationEnabled }}
   ROLLOUTS_CONTROLLER_INSTANCE_ID: {{ quote .Values.controller.rollouts.controllerInstanceID }}
   {{- end }}
+  MAX_CONCURRENT_CONTROL_FLOW_RECONCILES: {{ .Values.controller.reconcilers.controlFlowStages.maxConcurrentReconciles | default .Values.controller.reconcilers.maxConcurrentReconciles | quote }}
+  MAX_CONCURRENT_PROMOTION_RECONCILES: {{ .Values.controller.reconcilers.promotions.maxConcurrentReconciles | default .Values.controller.reconcilers.maxConcurrentReconciles | quote }}
+  MAX_CONCURRENT_STAGE_RECONCILES: {{ .Values.controller.reconcilers.stages.maxConcurrentReconciles | default .Values.controller.reconcilers.maxConcurrentReconciles | quote }}
+  MAX_CONCURRENT_WAREHOUSE_RECONCILES: {{ .Values.controller.reconcilers.warehouses.maxConcurrentReconciles | default .Values.controller.reconcilers.maxConcurrentReconciles | quote }}
 {{- end }}
diff --git a/charts/kargo/templates/controller/deployment.yaml b/charts/kargo/templates/controller/deployment.yaml
@@ -58,9 +58,19 @@ spec:
         image: {{ include "kargo.image" . }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         command: ["/usr/local/bin/kargo", "controller"]
-        {{- with (concat .Values.global.env .Values.controller.env) }}
         env:
-          {{- toYaml . | nindent 8 }}
+        - name: GOMEMLIMIT
+          valueFrom:
+            resourceFieldRef:
+              containerName: controller
+              resource: limits.memory
+        - name: GOMAXPROCS
+          valueFrom:
+            resourceFieldRef:
+              containerName: controller
+              resource: limits.cpu
+        {{- with (concat .Values.global.env .Values.controller.env) }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
         envFrom:
         - configMapRef:

diff --git a/charts/kargo/templates/dex-server/deployment.yaml b/charts/kargo/templates/dex-server/deployment.yaml
@@ -42,13 +42,23 @@ spec:
         imagePullPolicy: {{ .Values.api.oidc.dex.image.pullPolicy }}
         command: ["dex", "serve"]
         args: ["/etc/dex/config.yaml"]
-        {{- with (concat .Values.global.env .Values.api.oidc.dex.env) }}
         env:
-          {{- toYaml . | nindent 8 }}
+        - name: GOMEMLIMIT
+          valueFrom:
+            resourceFieldRef:
+              containerName: dex-server
+              resource: limits.memory
+        - name: GOMAXPROCS
+          valueFrom:
+            resourceFieldRef:
+              containerName: dex-server
+              resource: limits.cpu
+        {{- with (concat .Values.global.env .Values.api.oidc.dex.env) }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
         {{- with (concat .Values.global.envFrom .Values.api.oidc.dex.envFrom) }}
         envFrom:
-          {{- toYaml . | nindent 8 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
         volumeMounts:
         - mountPath: /etc/dex

diff --git a/charts/kargo/templates/garbage-collector/cron-job.yaml b/charts/kargo/templates/garbage-collector/cron-job.yaml
@@ -57,9 +57,19 @@ spec:
             image: {{ include "kargo.image" . }}
             imagePullPolicy: {{ .Values.image.pullPolicy }}
             command: ["/usr/local/bin/kargo", "garbage-collector"]
-            {{- with (concat .Values.global.env .Values.garbageCollector.env) }}
             env:
-              {{- toYaml . | nindent 12 }}
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  containerName: garbage-collector
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  containerName: garbage-collector
+                  resource: limits.cpu
+            {{- with (concat .Values.global.env .Values.garbageCollector.env) }}
+            {{- toYaml . | nindent 12 }}
             {{- end }}
             envFrom:
             - configMapRef:

diff --git a/charts/kargo/templates/management-controller/configmap.yaml b/charts/kargo/templates/management-controller/configmap.yaml
@@ -16,4 +16,7 @@ data:
   {{- if .Values.kubeconfigSecrets.kargo }}
   KUBECONFIG: /etc/kargo/kubeconfigs/kubeconfig.yaml
   {{- end }}
+  MAX_CONCURRENT_NAMESPACE_RECONCILES: {{ .Values.managementController.reconcilers.namespaces.maxConcurrentReconciles | default .Values.managementController.reconcilers.maxConcurrentReconciles | quote }}
+  MAX_CONCURRENT_PROJECT_RECONCILES: {{ .Values.managementController.reconcilers.projects.maxConcurrentReconciles | default .Values.managementController.reconcilers.maxConcurrentReconciles | quote }}
+  MAX_CONCURRENT_SERVICE_ACCOUNT_RECONCILES: {{ .Values.managementController.reconcilers.serviceAccounts.maxConcurrentReconciles | default .Values.managementController.reconcilers.maxConcurrentReconciles | quote }}
 {{- end }}
diff --git a/charts/kargo/templates/management-controller/deployment.yaml b/charts/kargo/templates/management-controller/deployment.yaml
@@ -58,9 +58,19 @@ spec:
         image: {{ include "kargo.image" . }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         command: ["/usr/local/bin/kargo", "management-controller"]
-        {{- with (concat .Values.global.env .Values.managementController.env) }}
         env:
-          {{- toYaml . | nindent 8 }}
+        - name: GOMEMLIMIT
+          valueFrom:
+            resourceFieldRef:
+              containerName: management-controller
+              resource: limits.memory
+        - name: GOMAXPROCS
+          valueFrom:
+            resourceFieldRef:
+              containerName: management-controller
+              resource: limits.cpu
+        {{- with (concat .Values.global.env .Values.managementController.env) }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
         envFrom:
         - configMapRef:

diff --git a/charts/kargo/templates/webhooks-server/deployment.yaml b/charts/kargo/templates/webhooks-server/deployment.yaml
@@ -58,9 +58,19 @@ spec:
         image: {{ include "kargo.image" . }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         command: ["/usr/local/bin/kargo", "webhooks-server"]
-        {{- with (concat .Values.global.env .Values.webhooksServer.env) }}
         env:
-          {{- toYaml . | nindent 8 }}
+        - name: GOMEMLIMIT
+          valueFrom:
+            resourceFieldRef:
+              containerName: webhooks-server
+              resource: limits.memory
+        - name: GOMAXPROCS
+          valueFrom:
+            resourceFieldRef:
+              containerName: webhooks-server
+              resource: limits.cpu
+        {{- with (concat .Values.global.env .Values.webhooksServer.env) }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
         envFrom:
         - configMapRef:

diff --git a/charts/kargo/values.yaml b/charts/kargo/values.yaml
@@ -377,6 +377,23 @@ controller:
     ## @param controller.globalCredentials.namespaces List of namespaces to look for shared credentials. Note that as of v1.0.0, the Kargo controller does not have cluster-wide access to Secrets. The controller receives read-only permission for Secrets on a per-Project basis as Projects are created. If you designate some namespaces as homes for "global" credentials, you will need to manually grant the controller permission to read Secrets in those namespaces.
     namespaces: []
 
+  ## Reconciler-specific settings
+  reconcilers:
+    ## @param controller.reconcilers.maxConcurrentReconciles specifies the maximum number of resources EACH of the controller's reconcilers can reconcile concurrently. This setting may also be overridden on a per-reconciler basis.
+    maxConcurrentReconciles: 4
+    controlFlowStages:
+      ## @param controller.reconcilers.controlFlowStages.maxConcurrentReconciles optionally overrides the maximum number of control flow Stage resources the controller can reconcile concurrently.
+      maxConcurrentReconciles:
+    promotions:
+      ## @param controller.reconcilers.promotions.maxConcurrentReconciles optionally overrides the maximum number of Promotion resources the controller can reconcile concurrently.
+      maxConcurrentReconciles:
+    stages:
+      ## @param controller.reconcilers.stages.maxConcurrentReconciles optionally overrides the maximum number of (non-control flow) Stage resources the controller can reconcile concurrently.
+      maxConcurrentReconciles:
+    warehouses:
+      ## @param controller.reconcilers.warehouses.maxConcurrentReconciles optionally overrides the maximum number of Warehouse resources the controller can reconcile concurrently.
+      maxConcurrentReconciles:
+
   gitClient:
     ## @param controller.gitClient.name Specifies the name of the Kargo controller (used when authoring Git commits).
     name: "Kargo"
@@ -465,6 +482,20 @@ managementController:
   ## @param managementController.podAnnotations Optional annotations to add to pods. Merges with `global.podAnnotations`, allowing you to override or add to the global annotations.
   podAnnotations: {}
 
+  ## Reconciler-specific settings
+  reconcilers:
+    ## @param managementController.reconcilers.maxConcurrentReconciles specifies the maximum number of resources EACH of the management controller's reconcilers can reconcile concurrently. This setting may also be overridden on a per-reconciler basis.
+    maxConcurrentReconciles: 4
+    namespaces:
+      ## @param managementController.reconcilers.namespaces.maxConcurrentReconciles optionally overrides the maximum number of Namespace resources the management controller can reconcile concurrently.
+      maxConcurrentReconciles:
+    projects:
+      ## @param managementController.reconcilers.projects.maxConcurrentReconciles optionally overrides the maximum number of Project resources the management controller can reconcile concurrently.
+      maxConcurrentReconciles:
+    serviceAccounts:
+      ## @param managementController.reconcilers.serviceAccounts.maxConcurrentReconciles optionally overrides the maximum number of ServiceAccount resources the management controller can reconcile concurrently.
+      maxConcurrentReconciles:
+
   ## @param managementController.securityContext Security context for management controller pods. Defaults to `global.securityContext`.
   securityContext: {}
 

diff --git a/cmd/controlplane/api.go b/cmd/controlplane/api.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"runtime"
 
 	"github.com/spf13/cobra"
 
@@ -61,6 +62,8 @@ func (o *apiOptions) run(ctx context.Context) error {
 		"Starting Kargo API Server",
 		"version", version.Version,
 		"commit", version.GitCommit,
+		"GOMAXPROCS", runtime.GOMAXPROCS(0),
+		"GOMEMLIMIT", os.GetEnv("GOMEMLIMIT", ""),
 	)
 
 	serverCfg := config.ServerConfigFromEnv()

diff --git a/cmd/controlplane/controller.go b/cmd/controlplane/controller.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	stdruntime "runtime"
 	"sync"
 
 	"github.com/spf13/cobra"
@@ -84,16 +85,18 @@ func (o *controllerOptions) run(ctx context.Context) error {
 	startupLogger := o.Logger.WithValues(
 		"version", version.Version,
 		"commit", version.GitCommit,
+		"GOMAXPROCS", stdruntime.GOMAXPROCS(0),
+		"GOMEMLIMIT", os.GetEnv("GOMEMLIMIT", ""),
 	)
 	if o.ShardName != "" {
 		startupLogger = startupLogger.WithValues("shard", o.ShardName)
 	}
 	startupLogger.Info("Starting Kargo Controller")
 
-	promotionsReconcilerCfg := promotions.ReconcilerConfigFromEnv()
-	stagesReconcilerCfg := stages.ReconcilerConfigFromEnv()
-
-	kargoMgr, stagesReconcilerCfg, err := o.setupKargoManager(ctx, stagesReconcilerCfg)
+	kargoMgr, stagesReconcilerCfg, err := o.setupKargoManager(
+		ctx,
+		stages.ReconcilerConfigFromEnv(),
+	)
 	if err != nil {
 		return fmt.Errorf("error initializing Kargo controller manager: %w", err)
 	}
@@ -114,7 +117,6 @@ func (o *controllerOptions) run(ctx context.Context) error {
 		kargoMgr,
 		argocdMgr,
 		credentialsDB,
-		promotionsReconcilerCfg,
 		stagesReconcilerCfg,
 	); err != nil {
 		return fmt.Errorf("error setting up reconcilers: %w", err)
@@ -285,7 +287,6 @@ func (o *controllerOptions) setupReconcilers(
 	ctx context.Context,
 	kargoMgr, argocdMgr manager.Manager,
 	credentialsDB credentials.Database,
-	promotionsReconcilerCfg promotions.ReconcilerConfig,
 	stagesReconcilerCfg stages.ReconcilerConfig,
 ) error {
 	var argoCDClient client.Client
@@ -301,7 +302,7 @@ func (o *controllerOptions) setupReconcilers(
 		kargoMgr,
 		argocdMgr,
 		directivesEngine,
-		promotionsReconcilerCfg,
+		promotions.ReconcilerConfigFromEnv(),
 	); err != nil {
 		return fmt.Errorf("error setting up Promotions reconciler: %w", err)
 	}
@@ -326,9 +327,10 @@ func (o *controllerOptions) setupReconcilers(
 	}
 
 	if err := warehouses.SetupReconcilerWithManager(
+		ctx,
 		kargoMgr,
 		credentialsDB,
-		o.ShardName,
+		warehouses.ReconcilerConfigFromEnv(),
 	); err != nil {
 		return fmt.Errorf("error setting up Warehouses reconciler: %w", err)
 	}

diff --git a/cmd/controlplane/garbage_collector.go b/cmd/controlplane/garbage_collector.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	stdruntime "runtime"
 
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
@@ -63,6 +64,8 @@ func (o *garbageCollectorOptions) run(ctx context.Context) error {
 		"Starting Kargo Garbage Collector",
 		"version", version.Version,
 		"commit", version.GitCommit,
+		"GOMAXPROCS", stdruntime.GOMAXPROCS(0),
+		"GOMEMLIMIT", os.GetEnv("GOMEMLIMIT", ""),
 	)
 
 	mgr, err := o.setupManager(ctx)

diff --git a/cmd/controlplane/management_controller.go b/cmd/controlplane/management_controller.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	stdruntime "runtime"
 
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
@@ -72,18 +73,25 @@ func (o *managementControllerOptions) run(ctx context.Context) error {
 		"Starting Kargo Management Controller",
 		"version", version.Version,
 		"commit", version.GitCommit,
+		"GOMAXPROCS", stdruntime.GOMAXPROCS(0),
+		"GOMEMLIMIT", os.GetEnv("GOMEMLIMIT", ""),
 	)
 
 	kargoMgr, err := o.setupManager(ctx)
 	if err != nil {
 		return fmt.Errorf("error initializing Kargo controller manager: %w", err)
 	}
 
-	if err := namespaces.SetupReconcilerWithManager(kargoMgr); err != nil {
+	if err := namespaces.SetupReconcilerWithManager(
+		ctx,
+		kargoMgr,
+		namespaces.ReconcilerConfigFromEnv(),
+	); err != nil {
 		return fmt.Errorf("error setting up Namespaces reconciler: %w", err)
 	}
 
 	if err := projects.SetupReconcilerWithManager(
+		ctx,
 		kargoMgr,
 		projects.ReconcilerConfigFromEnv(),
 	); err != nil {
@@ -92,6 +100,7 @@ func (o *managementControllerOptions) run(ctx context.Context) error {
 
 	if o.ManageControllerRoleBindings {
 		if err := serviceaccounts.SetupReconcilerWithManager(
+			ctx,
 			kargoMgr,
 			serviceaccounts.ReconcilerConfigFromEnv(),
 		); err != nil {

diff --git a/cmd/controlplane/webhooks.go b/cmd/controlplane/webhooks.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	stdruntime "runtime"
 
 	"github.com/spf13/cobra"
 	authzv1 "k8s.io/api/authorization/v1"
@@ -68,6 +69,8 @@ func (o *webhooksServerOptions) run(ctx context.Context) error {
 		"Starting Kargo Webhooks Server",
 		"version", version.Version,
 		"commit", version.GitCommit,
+		"GOMAXPROCS", stdruntime.GOMAXPROCS(0),
+		"GOMEMLIMIT", os.GetEnv("GOMEMLIMIT", ""),
 	)
 
 	webhookCfg := libWebhook.ConfigFromEnv()