Skip to content

Commit

Permalink
[FEATURE] - Executor Resource Requests & Limits (#1501)
Browse files Browse the repository at this point in the history 
* [FEAT] - Executor Resource Requests & Limits

Currently consumers are not permitted to set the executor resource requests/limits - this has created an error in the past, due to higher than expected requirements, hence it was bumped to 1Gi. For certain modules though this may not be enough, hence the feature to support controling the fields from the controller arguments

* feat: ensure the inputs are valid quantities

* fix: adjusting due to formatting issues

* feat: removing the limitation by default and placing the decision on the administrator as to limits
  • Loading branch information
@gambol99
gambol99 authored Aug 31, 2024
1 parent d94143d commit b488ca7
Show file tree
Hide file tree
Showing 8 changed files with 124 additions and 54 deletions.
4 changes: 4 additions & 0 deletions cmd/controller/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ func main() {
flags.IntVar(&config.MetricsPort, "metrics-port", 9090, "The port the metric endpoint binds to")
flags.IntVar(&config.WebhookPort, "webhooks-port", 10081, "The port the webhook endpoint binds to")
flags.StringSliceVar(&config.ExecutorSecrets, "executor-secret", []string{}, "Name of a secret in controller namespace which should be added to the job")
flags.StringVar(&config.ExecutorMemoryRequest, "executor-memory-request", "32Mi", "The default memory request for the executor container")
flags.StringVar(&config.ExecutorMemoryLimit, "executor-memory-limit", "", "The default memory limit for the executor container (default is no limit)")
flags.StringVar(&config.ExecutorCPURequest, "executor-cpu-request", "5m", "The default CPU request for the executor container")
flags.StringVar(&config.ExecutorCPULimit, "executor-cpu-limit", "", "The default CPU limit for the executor container (default is no limit)")
flags.StringVar(&config.BackendTemplate, "backend-template", "", "Name of secret in the controller namespace containing a template for the terraform state")
flags.StringVar(&config.ExecutorImage, "executor-image", fmt.Sprintf("ghcr.io/appvia/terranetes-executor:%s", version.Version), "The image to use for the executor")
flags.StringVar(&config.InfracostsImage, "infracost-image", "infracosts/infracost:latest", "The image to use for the infracosts")
Expand Down
14 changes: 10 additions & 4 deletions pkg/assets/job.yaml.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,12 +259,18 @@ spec:
optional: false
{{- end }}
resources:
{{- if or (ne .DefaultExecutorCPULimit "") (ne .DefaultExecutorMemoryLimit "") }}
limits:
cpu: 1
memory: 1Gi
{{- if ne .DefaultExecutorCPULimit "" }}
cpu: {{ .DefaultExecutorCPULimit }}
{{- end }}
{{- if ne .DefaultExecutorMemoryLimit "" }}
memory: {{ .DefaultExecutorMemoryLimit }}
{{- end }}
{{- end }}
requests:
cpu: 5m
memory: 32Mi
cpu: {{ .DefaultExecutorCPURequest }}
memory: {{ .DefaultExecutorMemoryRequest }}
securityContext:
capabilities:
drop: [ALL]
Expand Down
24 changes: 24 additions & 0 deletions pkg/controller/configuration/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
log "github.com/sirupsen/logrus"
batchv1 "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
cache "k8s.io/client-go/tools/cache"
Expand Down Expand Up @@ -89,6 +90,14 @@ type Controller struct {
// executors job everytime - these are configured by the platform team on the
// cli options
ExecutorSecrets []string
// DefaultExecutorMemoryRequest is the default memory request for the executor container
DefaultExecutorMemoryRequest string
// DefaultExecutorMemoryLimit is the default memory limit for the executor container
DefaultExecutorMemoryLimit string
// DefaultExecutorCPURequest is the default CPU request for the executor container
DefaultExecutorCPURequest string
// DefaultExecutorCPULimit is the default CPU limit for the executor container
DefaultExecutorCPULimit string
// InfracostsImage is the image to use for all infracost jobs
InfracostsImage string
// InfracostsSecretName is the name of the secret containing the api and token
Expand Down Expand Up @@ -161,6 +170,21 @@ func (c *Controller) Add(mgr manager.Manager) error {
"terraform_image": c.TerraformImage,
}).Info("adding the configuration controller")

// @step: ensure the resource limits are valid
for _, c := range []string{
c.DefaultExecutorCPULimit,
c.DefaultExecutorCPURequest,
c.DefaultExecutorMemoryLimit,
c.DefaultExecutorMemoryRequest,
} {
if c == "" {
continue
}
if _, err := resource.ParseQuantity(c); err != nil {
return fmt.Errorf("invalid resource quantity: %q, error: %w", c, err)
}
}

switch {
case c.ControllerNamespace == "":
return errors.New("job namespace is required")
Expand Down
28 changes: 16 additions & 12 deletions pkg/controller/configuration/ensure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -815,18 +815,22 @@ func (c *Controller) ensureTerraformPlan(configuration *terraformv1alpha1.Config
terraformv1alpha1.RetryAnnotation: configuration.GetAnnotations()[terraformv1alpha1.RetryAnnotation],
terraformv1alpha1.JobTemplateHashLabel: state.jobTemplateHash,
}),
BackoffLimit: c.BackoffLimit,
EnableInfraCosts: c.EnableInfracosts,
ExecutorImage: c.ExecutorImage,
ExecutorSecrets: c.ExecutorSecrets,
InfracostsImage: c.InfracostsImage,
InfracostsSecret: c.InfracostsSecretName,
Namespace: c.ControllerNamespace,
PolicyConstraint: state.checkovConstraint,
PolicyImage: c.PolicyImage,
SaveTerraformState: saveState,
Template: state.jobTemplate,
TerraformImage: GetTerraformImage(configuration, c.TerraformImage),
BackoffLimit: c.BackoffLimit,
DefaultExecutorCPULimit: c.DefaultExecutorCPULimit,
DefaultExecutorCPURequest: c.DefaultExecutorCPURequest,
DefaultExecutorMemoryLimit: c.DefaultExecutorMemoryLimit,
DefaultExecutorMemoryRequest: c.DefaultExecutorMemoryRequest,
EnableInfraCosts: c.EnableInfracosts,
ExecutorImage: c.ExecutorImage,
ExecutorSecrets: c.ExecutorSecrets,
InfracostsImage: c.InfracostsImage,
InfracostsSecret: c.InfracostsSecretName,
Namespace: c.ControllerNamespace,
PolicyConstraint: state.checkovConstraint,
PolicyImage: c.PolicyImage,
SaveTerraformState: saveState,
Template: state.jobTemplate,
TerraformImage: GetTerraformImage(configuration, c.TerraformImage),
}

// @step: use the options to generate the job
Expand Down
54 changes: 31 additions & 23 deletions pkg/controller/configuration/reconcile_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,18 +56,22 @@ func TestController(t *testing.T) {
func makeFakeController(cc client.Client) *Controller {
recorder := &controllertests.FakeRecorder{}
ctrl := &Controller{
cc: cc,
kc: kfake.NewSimpleClientset(),
cache: cache.New(5*time.Minute, 10*time.Minute),
recorder: recorder,
BackoffLimit: 2,
EnableInfracosts: false,
EnableWatchers: true,
ExecutorImage: "ghcr.io/appvia/terranetes-executor",
InfracostsImage: "infracosts/infracost:latest",
ControllerNamespace: "terraform-system",
PolicyImage: "bridgecrew/checkov:2.0.1140",
TerraformImage: "hashicorp/terraform:1.1.9",
cc: cc,
kc: kfake.NewSimpleClientset(),
cache: cache.New(5*time.Minute, 10*time.Minute),
recorder: recorder,
BackoffLimit: 2,
ControllerNamespace: "terraform-system",
DefaultExecutorCPULimit: "1",
DefaultExecutorCPURequest: "5m",
DefaultExecutorMemoryLimit: "1Gi",
DefaultExecutorMemoryRequest: "32Mi",
EnableInfracosts: false,
EnableWatchers: true,
ExecutorImage: "ghcr.io/appvia/terranetes-executor",
InfracostsImage: "infracosts/infracost:latest",
PolicyImage: "bridgecrew/checkov:2.0.1140",
TerraformImage: "hashicorp/terraform:1.1.9",
}

return ctrl
Expand Down Expand Up @@ -928,17 +932,21 @@ var _ = Describe("Configuration Controller", func() {

recorder = &controllertests.FakeRecorder{}
ctrl = &Controller{
cc: cc,
kc: kfake.NewSimpleClientset(),
cache: cache.New(5*time.Minute, 10*time.Minute),
recorder: recorder,
EnableInfracosts: false,
EnableWatchers: true,
ExecutorImage: "ghcr.io/appvia/terranetes-executor",
InfracostsImage: "infracosts/infracost:latest",
ControllerNamespace: "default",
PolicyImage: "bridgecrew/checkov:2.0.1140",
TerraformImage: "hashicorp/terraform:1.1.9",
cc: cc,
kc: kfake.NewSimpleClientset(),
cache: cache.New(5*time.Minute, 10*time.Minute),
recorder: recorder,
DefaultExecutorCPULimit: "1",
DefaultExecutorCPURequest: "5m",
DefaultExecutorMemoryLimit: "1Gi",
DefaultExecutorMemoryRequest: "32Mi",
EnableInfracosts: false,
EnableWatchers: true,
ExecutorImage: "ghcr.io/appvia/terranetes-executor",
InfracostsImage: "infracosts/infracost:latest",
ControllerNamespace: "default",
PolicyImage: "bridgecrew/checkov:2.0.1140",
TerraformImage: "hashicorp/terraform:1.1.9",
}
ctrl.cache.SetDefault(cfgNamespace, fixtures.NewNamespace(cfgNamespace))
}
Expand Down
34 changes: 19 additions & 15 deletions pkg/server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,21 +171,25 @@ func New(cfg *rest.Config, config Config) (*Server, error) {

// @step: ensure the configuration controller is enabled
if err := (&configuration.Controller{
BackendTemplate: config.BackendTemplate,
BackoffLimit: config.BackoffLimit,
ControllerJobLabels: jobLabels,
ControllerNamespace: config.Namespace,
EnableInfracosts: (config.InfracostsSecretName != ""),
EnableTerraformVersions: config.EnableTerraformVersions,
EnableWatchers: config.EnableWatchers,
EnableWebhooks: config.EnableWebhooks,
ExecutorImage: config.ExecutorImage,
ExecutorSecrets: config.ExecutorSecrets,
InfracostsImage: config.InfracostsImage,
InfracostsSecretName: config.InfracostsSecretName,
JobTemplate: config.JobTemplate,
PolicyImage: config.PolicyImage,
TerraformImage: config.TerraformImage,
BackendTemplate: config.BackendTemplate,
BackoffLimit: config.BackoffLimit,
ControllerJobLabels: jobLabels,
ControllerNamespace: config.Namespace,
DefaultExecutorCPULimit: config.ExecutorCPULimit,
DefaultExecutorCPURequest: config.ExecutorCPURequest,
DefaultExecutorMemoryLimit: config.ExecutorMemoryLimit,
DefaultExecutorMemoryRequest: config.ExecutorMemoryRequest,
EnableInfracosts: (config.InfracostsSecretName != ""),
EnableTerraformVersions: config.EnableTerraformVersions,
EnableWatchers: config.EnableWatchers,
EnableWebhooks: config.EnableWebhooks,
ExecutorImage: config.ExecutorImage,
ExecutorSecrets: config.ExecutorSecrets,
InfracostsImage: config.InfracostsImage,
InfracostsSecretName: config.InfracostsSecretName,
JobTemplate: config.JobTemplate,
PolicyImage: config.PolicyImage,
TerraformImage: config.TerraformImage,
}).Add(mgr); err != nil {
return nil, fmt.Errorf("failed to create the configuration controller, error: %w", err)
}
Expand Down
8 changes: 8 additions & 0 deletions pkg/server/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,14 @@ type Config struct {
EnableWebhookPrefix bool
// ExecutorImage is the image to use for the executor
ExecutorImage string
// ExecutorMemoryRequest is the memory request for the executor container (e.g. 1Gi, 2Gi)
ExecutorMemoryRequest string
// ExecutorMemoryLimit is the memory limit for the executor container (e.g. 1Gi, 2Gi)
ExecutorMemoryLimit string
// ExecutorCPURequest is the CPU request for the executor container (e.g. 1, 2)
ExecutorCPURequest string
// ExecutorCPULimit is the CPU limit for the executor container (e.g. 1, 2)
ExecutorCPULimit string
// ExecutorSecrets is a list of additional secrets to be added to the executor
ExecutorSecrets []string
// InfracostsSecretName is the name of the secret that contains the cost token and endpoint
Expand Down
12 changes: 12 additions & 0 deletions pkg/utils/jobs/jobs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ type Options struct {
// BackoffLimit is the number of times we are willing to allow a job to fail
// before we give up
BackoffLimit int
// DefaultExecutorMemoryRequest is the default memory request for the executor
DefaultExecutorMemoryRequest string
// DefaultExecutorMemoryLimit is the default memory limit for the executor
DefaultExecutorMemoryLimit string
// DefaultExecutorCPURequest is the default CPU request for the executor
DefaultExecutorCPURequest string
// DefaultExecutorCPULimit is the default CPU limit for the executor
DefaultExecutorCPULimit string
// EnableInfraCosts is the flag to enable cost analysis
EnableInfraCosts bool
// ExecutorImage is the image to use for the terraform jobs
Expand Down Expand Up @@ -206,6 +214,10 @@ func (r *Render) createTerraformFromTemplate(options Options, stage string) (*ba
terraformv1alpha1.ConfigurationStageLabel: stage,
terraformv1alpha1.ConfigurationUIDLabel: string(r.configuration.GetUID()),
}),
"DefaultExecutorMemoryRequest": options.DefaultExecutorMemoryRequest,
"DefaultExecutorMemoryLimit": options.DefaultExecutorMemoryLimit,
"DefaultExecutorCPURequest": options.DefaultExecutorCPURequest,
"DefaultExecutorCPULimit": options.DefaultExecutorCPULimit,
"Provider": map[string]interface{}{
"Name": r.provider.Name,
"Namespace": r.provider.Namespace,
Expand Down

0 comments on commit b488ca7

Please sign in to comment.