TEP-0135: Introduce coscheduling feature flags

Part of [#6740][#6740]. [TEP-0135][tep-0135] introduces a feature that allows a cluster operator
to ensure that all of a PipelineRun's pods are scheduled to the same node.

This commit introduces a new feature flag `coscheduling` which works together with the `disable-affinity-assistant` feature flag
to determine the Affinity Assistant behavior. The usage of the new feature flag will be added in the follow-up PRs.

The details of the `coscheduling` feature flag can be found in the [Configuration][configuration] section of TEP-0135.
The details of the `disable-affinity-assistant` feature flag can be found in the [Upgrade and Migration Strategy][strategy] section of TEP-0135.

NOTE: this feature is WIP, please do not use on this feature.

/kind feature

[#6740]: #6740
[tep-0135]: https://github.com/tektoncd/community/blob/main/teps/0135-coscheduling-pipelinerun-pods.md
[configuration]: https://github.com/tektoncd/community/blob/main/teps/0135-coscheduling-pipelinerun-pods.md#configuration
[strategy]: https://github.com/tektoncd/community/blob/main/teps/0135-coscheduling-pipelinerun-pods.md#configuration

config/config-feature-flags.yaml

@@ -30,6 +30,18 @@ data:
   # https://github.com/tektoncd/pipeline/blob/main/docs/workspaces.md#affinity-assistant-and-specifying-workspace-order-in-a-pipeline
   # or https://github.com/tektoncd/pipeline/pull/2630 for more info.
   disable-affinity-assistant: "false"
+  # Setting this flag will determine how PipelineRun Pods are scheduled with Affinity Assistant.
+  # Acceptable values are "coschedule-workspaces" (default), "coschedule-pipelineruns", "isolate-pipelineruns", or "disabled" .
+  #
+  # Setting it to "coschedule-workspaces" will schedule all the taskruns sharing the same PVC-based workspace in a pipelinerun to the same node.
+  # Setting it to "coschedule-pipelineruns" will schedule all the taskruns in a pipelinerun to the same node.
+  # Setting it to "isolate-pipelineruns" will schedule all the taskruns in a pipelinerun to the same node, 
+  # and only allows one pipelinerun to run on a node at a time.
+  # Setting it to "disabled" will not apply any coscheduling policy.
+  #
+  # TODO: add links to documentation and migration strategy
+  # NOTE: this feature is still under development and not yet functional.
+  coscheduling: "coschedule-workspaces"
   # Setting this flag to "true" will prevent Tekton scanning attached
   # service accounts and injecting any credentials it finds into your
   # Steps.

pkg/apis/config/feature_flags.go

@@ -42,6 +42,14 @@ const (
 	// IgnoreNoMatchPolicy is the value used for "trusted-resources-verification-no-match-policy" to skip verification
 	// when no matching policies are found
 	IgnoreNoMatchPolicy = "ignore"
+	// CosheduleWorkspaces is the value used for "coscheduling" to coschedule PipelineRun Pods sharing the same PVC workspaces to the same node
+	CoscheduleWorkspaces = "coschedule-workspaces"
+	// CoshedulePipelineRuns is the value used for "coscheduling" to coschedule all PipelineRun Pods to the same node
+	CoschedulePipelineRuns = "coschedule-pipelineruns"
+	// CoscheduleIsolatePipelineRuns is the value used for "coscheduling" to coschedule all PipelineRun Pods to the same node, and only allows one PipelineRun to run on a node at a time
+	CoscheduleIsolatePipelineRuns = "isolate-pipelineruns"
+	// CoscheduleDisabled is the value used for "coscheduling" to disabled PipelineRun Pods coscheduling
+	CoscheduleDisabled = "disabled"
 	// ResultExtractionMethodTerminationMessage is the value used for "results-from" as a way to extract results from tasks using kubernetes termination message.
 	ResultExtractionMethodTerminationMessage = "termination-message"
 	// ResultExtractionMethodSidecarLogs is the value used for "results-from" as a way to extract results from tasks using sidecar logs.
@@ -78,6 +86,8 @@ const (
 	DefaultMaxResultSize = 4096
 	// DefaultSetSecurityContext is the default value for "set-security-context"
 	DefaultSetSecurityContext = false
+	// DefaultCoscheduling is the default value for coscheduling
+	DefaultCoscheduling = CoscheduleWorkspaces
 
 	disableAffinityAssistantKey         = "disable-affinity-assistant"
 	disableCredsInitKey                 = "disable-creds-init"
@@ -93,6 +103,7 @@ const (
 	resultExtractionMethod              = "results-from"
 	maxResultSize                       = "max-result-size"
 	setSecurityContextKey               = "set-security-context"
+	coschedulingKey                     = "coscheduling"
 )
 
 // DefaultFeatureFlags holds all the default configurations for the feature flags configmap.
@@ -123,6 +134,7 @@ type FeatureFlags struct {
 	ResultExtractionMethod    string
 	MaxResultSize             int
 	SetSecurityContext        bool
+	Coscheduling              string
 }
 
 // GetFeatureFlagsConfigName returns the name of the configmap containing all
@@ -190,6 +202,9 @@ func NewFeatureFlagsFromMap(cfgMap map[string]string) (*FeatureFlags, error) {
 		return nil, err
 	}
 
+	if err := setCoscheduling(cfgMap, DefaultCoscheduling, tc.DisableAffinityAssistant, &tc.Coscheduling); err != nil {
+		return nil, err
+	}
 	// Given that they are alpha features, Tekton Bundles and Custom Tasks should be switched on if
 	// enable-api-fields is "alpha". If enable-api-fields is not "alpha" then fall back to the value of
 	// each feature's individual flag.
@@ -222,6 +237,29 @@ func setEnabledAPIFields(cfgMap map[string]string, defaultValue string, feature
 	return nil
 }
 
+// setCoscheduling sets the "coscheduling" flag based on the content of a given map.
+// If the feature gate is invalid or incompatible with `disable-affinity-assistant`, then an error is returned.
+func setCoscheduling(cfgMap map[string]string, defaultValue string, disabledAffinityAssistant bool, feature *string) error {
+	value := defaultValue
+	if cfg, ok := cfgMap[coschedulingKey]; ok {
+		value = strings.ToLower(cfg)
+	}
+
+	switch value {
+	case CoscheduleDisabled, CoscheduleWorkspaces, CoschedulePipelineRuns, CoscheduleIsolatePipelineRuns:
+		// validate that "coscheduling" is compatible with "disable-affinity-assistant"
+		// "coscheduling" must be set to "coschedule-workspaces" when "disable-affinity-assistant" is false
+		if !disabledAffinityAssistant && value != CoscheduleWorkspaces {
+			return fmt.Errorf("coscheduling value %v is incompatible with %v setting to false", value, disableAffinityAssistantKey)
+		}
+		*feature = value
+	default:
+		return fmt.Errorf("invalid value for feature flag %q: %q", coschedulingKey, value)
+	}
+
+	return nil
+}
+
 // setEnforceNonFalsifiability sets the "enforce-nonfalsifiability" flag based on the content of a given map.
 // If the feature gate is invalid, then an error is returned.
 func setEnforceNonFalsifiability(cfgMap map[string]string, enableAPIFields string, feature *string) error {

pkg/apis/config/feature_flags_test.go

@@ -52,6 +52,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				ResultExtractionMethod:    config.DefaultResultExtractionMethod,
 				MaxResultSize:             config.DefaultMaxResultSize,
 				SetSecurityContext:        config.DefaultSetSecurityContext,
+				Coscheduling:              config.DefaultCoscheduling,
 			},
 			fileName: config.GetFeatureFlagsConfigName(),
 		},
@@ -70,6 +71,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				ResultExtractionMethod:           "termination-message",
 				MaxResultSize:                    4096,
 				SetSecurityContext:               true,
+				Coscheduling:                     config.CoscheduleDisabled,
 			},
 			fileName: "feature-flags-all-flags-set",
 		},
@@ -91,6 +93,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				ResultExtractionMethod:           config.DefaultResultExtractionMethod,
 				MaxResultSize:                    config.DefaultMaxResultSize,
 				SetSecurityContext:               config.DefaultSetSecurityContext,
+				Coscheduling:                     config.DefaultCoscheduling,
 			},
 			fileName: "feature-flags-enable-api-fields-overrides-bundles-and-custom-tasks",
 		},
@@ -110,6 +113,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				ResultExtractionMethod:           config.DefaultResultExtractionMethod,
 				MaxResultSize:                    config.DefaultMaxResultSize,
 				SetSecurityContext:               config.DefaultSetSecurityContext,
+				Coscheduling:                     config.DefaultCoscheduling,
 			},
 			fileName: "feature-flags-bundles-and-custom-tasks",
 		},
@@ -129,6 +133,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				ResultExtractionMethod:           config.DefaultResultExtractionMethod,
 				MaxResultSize:                    config.DefaultMaxResultSize,
 				SetSecurityContext:               config.DefaultSetSecurityContext,
+				Coscheduling:                     config.DefaultCoscheduling,
 			},
 			fileName: "feature-flags-beta-api-fields",
 		},
@@ -144,6 +149,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				ResultExtractionMethod:           config.DefaultResultExtractionMethod,
 				MaxResultSize:                    config.DefaultMaxResultSize,
 				SetSecurityContext:               config.DefaultSetSecurityContext,
+				Coscheduling:                     config.DefaultCoscheduling,
 			},
 			fileName: "feature-flags-enforce-nonfalsifiability-spire",
 		},
@@ -157,6 +163,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				ResultExtractionMethod:           config.ResultExtractionMethodSidecarLogs,
 				MaxResultSize:                    8192,
 				SetSecurityContext:               config.DefaultSetSecurityContext,
+				Coscheduling:                     config.DefaultCoscheduling,
 			},
 			fileName: "feature-flags-results-via-sidecar-logs",
 		},
@@ -188,6 +195,7 @@ func TestNewFeatureFlagsFromEmptyConfigMap(t *testing.T) {
 		ResultExtractionMethod:           config.DefaultResultExtractionMethod,
 		MaxResultSize:                    config.DefaultMaxResultSize,
 		SetSecurityContext:               config.DefaultSetSecurityContext,
+		Coscheduling:                     config.DefaultCoscheduling,
 	}
 	verifyConfigFileWithExpectedFeatureFlagsConfig(t, FeatureFlagsConfigEmptyName, expectedConfig)
 }
@@ -247,6 +255,12 @@ func TestNewFeatureFlagsConfigMapErrors(t *testing.T) {
 	}, {
 		fileName: "feature-flags-spire-with-stable",
 		want:     `"enforce-nonfalsifiability" can be set to non-default values ("spire") only in alpha`,
+	}, {
+		fileName: "feature-flags-invalid-coscheduling-affinity-assistant-comb",
+		want:     `coscheduling value coschedule-pipelineruns is incompatible with disable-affinity-assistant setting to false`,
+	}, {
+		fileName: "feature-flags-invalid-coscheduling",
+		want:     `invalid value for feature flag "coscheduling": "invalid"`,
 	}} {
 		t.Run(tc.fileName, func(t *testing.T) {
 			cm := test.ConfigMapFromTestFile(t, tc.fileName)

pkg/apis/config/testdata/feature-flags-all-flags-set.yaml

@@ -19,6 +19,7 @@ metadata:
   namespace: tekton-pipelines
 data:
   disable-affinity-assistant: "true"
+  coscheduling: "disabled"
   running-in-environment-with-injected-sidecars: "false"
   await-sidecar-readiness: "false"
   require-git-ssh-secret-known-hosts: "true"

pkg/apis/config/testdata/feature-flags-invalid-coscheduling-affinity-assistant-comb.yaml

@@ -0,0 +1,22 @@
+# Copyright 2023 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+data:
+  coscheduling: "coschedule-pipelineruns"
+  disable-affinity-assistant: "false"

pkg/apis/config/testdata/feature-flags-invalid-coscheduling.yaml

@@ -0,0 +1,21 @@
+# Copyright 2023 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+data:
+  coscheduling: "invalid"

pkg/apis/config/testing/featureflags.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2023 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"context"
+	"testing"
+
+	"github.com/tektoncd/pipeline/pkg/apis/config"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"knative.dev/pkg/logging"
+)
+
+// SetFeatureFlags sets the feature-flags ConfigMap values in an existing context (for use in testing)
+func SetFeatureFlags(ctx context.Context, t *testing.T, data map[string]string) context.Context {
+	t.Helper()
+	s := config.NewStore(logging.FromContext(ctx).Named("config-store"))
+	s.OnConfigChanged(&corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: config.GetFeatureFlagsConfigName(),
+		},
+		Data: data,
+	})
+	return s.ToContext(ctx)
+}

pkg/internal/affinityassistant/affinityassistant_types.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2023 The Tekton Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package affinityassistant
+
+import (
+	"context"
+
+	"github.com/tektoncd/pipeline/pkg/apis/config"
+)
+
+type AffinityAssitantBehavior string
+
+const (
+	AffinityAssistantDisabled                    = AffinityAssitantBehavior("AffinityAssistantDisabled")
+	AffinityAssistantPerWorkspace                = AffinityAssitantBehavior("AffinityAssistantPerWorkspace")
+	AffinityAssistantPerPipelineRun              = AffinityAssitantBehavior("AffinityAssistantPerPipelineRun")
+	AffinityAssistantPerPipelineRunWithIsolation = AffinityAssitantBehavior("AffinityAssistantPerPipelineRunWithIsolation")
+)
+
+// GetAffinityAssistantBehavior returns an AffinityAssitantBehavior based on the
+// combination of "disable-affinity-assistant" and "coscheduling" feature flags
+// TODO(#6740)(WIP): consume this function in the PipelineRun reconciler to determine Affinity Assistant behavior.
+func GetAffinityAssistantBehavior(ctx context.Context) AffinityAssitantBehavior {
+	cfg := config.FromContextOrDefaults(ctx)
+
+	// at this point, we have validated that "coscheduling" can only be "coschedule-workspaces"
+	// when "disable-affinity-assistant" is false
+	if !cfg.FeatureFlags.DisableAffinityAssistant {
+		return AffinityAssistantPerWorkspace
+	}
+
+	switch cfg.FeatureFlags.Coscheduling {
+	case config.CoschedulePipelineRuns:
+		return AffinityAssistantPerPipelineRun
+	case config.CoscheduleIsolatePipelineRuns:
+		return AffinityAssistantPerPipelineRunWithIsolation
+	case config.CoscheduleDisabled, config.CoscheduleWorkspaces:
+		return AffinityAssistantDisabled
+	}
+
+	return ""
+}