From e747ca07fccd370e8ec0f0aa7eb44641c2d9c77a Mon Sep 17 00:00:00 2001 From: ArkBriar Date: Wed, 30 Oct 2024 20:36:15 +0800 Subject: [PATCH] feat: support secret encryption key (#746) * feat: support setting private key for secret store Signed-off-by: arkbriar * Add unit tests for the validation webhook Signed-off-by: arkbriar * Refine the behaviours Signed-off-by: arkbriar * Add license header Signed-off-by: arkbriar --------- Signed-off-by: arkbriar --- .../v1alpha1/risingwave_secret_store.go | 45 ++++++ apis/risingwave/v1alpha1/risingwave_types.go | 3 + .../v1alpha1/zz_generated.deepcopy.go | 57 +++++++ ...ngwave.risingwavelabs.com_risingwaves.yaml | 32 ++++ config/risingwave-operator-test.yaml | 32 ++++ config/risingwave-operator.yaml | 32 ++++ docs/general/api.md | 144 ++++++++++++++++++ pkg/factory/envs/risingwave.go | 61 ++++---- pkg/factory/risingwave_object_factory.go | 29 ++++ ...isingwave_object_factory_predicate_test.go | 35 +++++ pkg/factory/risingwave_object_factory_test.go | 40 +++++ ...isingwave_object_factory_testcases_test.go | 54 ++++++- pkg/features/feature_manager.go | 12 +- pkg/features/feature_manager_test.go | 6 + pkg/utils/rand.go | 29 ++++ pkg/webhook/risingwave_mutating_webhook.go | 13 ++ .../risingwave_mutating_webhook_test.go | 3 + pkg/webhook/risingwave_validating_webhook.go | 37 +++++ .../risingwave_validating_webhook_test.go | 141 +++++++++++++++++ 19 files changed, 772 insertions(+), 33 deletions(-) create mode 100644 apis/risingwave/v1alpha1/risingwave_secret_store.go create mode 100644 pkg/utils/rand.go diff --git a/apis/risingwave/v1alpha1/risingwave_secret_store.go b/apis/risingwave/v1alpha1/risingwave_secret_store.go new file mode 100644 index 00000000..ccafd661 --- /dev/null +++ b/apis/risingwave/v1alpha1/risingwave_secret_store.go @@ -0,0 +1,45 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +// RisingWaveSecretStorePrivateKeySecretReference is a reference to a secret that contains a private key. +type RisingWaveSecretStorePrivateKeySecretReference struct { + // Name is the name of the secret. + Name string `json:"name"` + + // Key is the key in the secret that contains the private key. + Key string `json:"key"` +} + +// RisingWaveSecretStorePrivateKey is a private key that can be stored in a secret or directly in the resource. +type RisingWaveSecretStorePrivateKey struct { + // Value is the private key. It must be a 128-bit key encoded in hex. If this is set, SecretRef must be nil. + // When the feature gate RandomSecretStorePrivateKey is enabled and neither is set, the private key will be + // generated randomly. + // +kubebuilder:validation:Pattern="^[0-9a-f]{32}$" + // +optional + Value *string `json:"value,omitempty"` + + // SecretRef is a reference to a secret that contains the private key. If this is set, Value must be nil. + // Note that the value in the secret must be a 128-bit key encoded in hex. + // +optional + SecretRef *RisingWaveSecretStorePrivateKeySecretReference `json:"secretRef,omitempty"` +} + +// RisingWaveSecretStore is the configuration of the secret store. +type RisingWaveSecretStore struct { + // PrivateKey is the private key used to encrypt and decrypt the secrets. + PrivateKey RisingWaveSecretStorePrivateKey `json:"privateKey,omitempty"` +} diff --git a/apis/risingwave/v1alpha1/risingwave_types.go b/apis/risingwave/v1alpha1/risingwave_types.go index 9712ade6..30f24f8f 100644 --- a/apis/risingwave/v1alpha1/risingwave_types.go +++ b/apis/risingwave/v1alpha1/risingwave_types.go @@ -148,6 +148,9 @@ type RisingWaveSpec struct { // LicenseKey to enable paid features of RisingWave. LicenseKey *RisingWaveLicenseKey `json:"licenseKey,omitempty"` + + // SecretStore is the configuration of the secret store. + SecretStore RisingWaveSecretStore `json:"secretStore,omitempty"` } // ComponentGroupReplicasStatus are the running status of Pods in group. diff --git a/apis/risingwave/v1alpha1/zz_generated.deepcopy.go b/apis/risingwave/v1alpha1/zz_generated.deepcopy.go index 1374b0b4..1515097e 100644 --- a/apis/risingwave/v1alpha1/zz_generated.deepcopy.go +++ b/apis/risingwave/v1alpha1/zz_generated.deepcopy.go @@ -1210,6 +1210,62 @@ func (in *RisingWaveScaleViewTargetRef) DeepCopy() *RisingWaveScaleViewTargetRef return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RisingWaveSecretStore) DeepCopyInto(out *RisingWaveSecretStore) { + *out = *in + in.PrivateKey.DeepCopyInto(&out.PrivateKey) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RisingWaveSecretStore. +func (in *RisingWaveSecretStore) DeepCopy() *RisingWaveSecretStore { + if in == nil { + return nil + } + out := new(RisingWaveSecretStore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RisingWaveSecretStorePrivateKey) DeepCopyInto(out *RisingWaveSecretStorePrivateKey) { + *out = *in + if in.Value != nil { + in, out := &in.Value, &out.Value + *out = new(string) + **out = **in + } + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(RisingWaveSecretStorePrivateKeySecretReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RisingWaveSecretStorePrivateKey. +func (in *RisingWaveSecretStorePrivateKey) DeepCopy() *RisingWaveSecretStorePrivateKey { + if in == nil { + return nil + } + out := new(RisingWaveSecretStorePrivateKey) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RisingWaveSecretStorePrivateKeySecretReference) DeepCopyInto(out *RisingWaveSecretStorePrivateKeySecretReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RisingWaveSecretStorePrivateKeySecretReference. +func (in *RisingWaveSecretStorePrivateKeySecretReference) DeepCopy() *RisingWaveSecretStorePrivateKeySecretReference { + if in == nil { + return nil + } + out := new(RisingWaveSecretStorePrivateKeySecretReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RisingWaveSpec) DeepCopyInto(out *RisingWaveSpec) { *out = *in @@ -1258,6 +1314,7 @@ func (in *RisingWaveSpec) DeepCopyInto(out *RisingWaveSpec) { *out = new(RisingWaveLicenseKey) (*in).DeepCopyInto(*out) } + in.SecretStore.DeepCopyInto(&out.SecretStore) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RisingWaveSpec. diff --git a/config/crd/bases/risingwave.risingwavelabs.com_risingwaves.yaml b/config/crd/bases/risingwave.risingwavelabs.com_risingwaves.yaml index 76d54997..8a1c0ee8 100644 --- a/config/crd/bases/risingwave.risingwavelabs.com_risingwaves.yaml +++ b/config/crd/bases/risingwave.risingwavelabs.com_risingwaves.yaml @@ -30510,6 +30510,38 @@ spec: - path type: object type: object + secretStore: + description: SecretStore is the configuration of the secret store. + properties: + privateKey: + description: PrivateKey is the private key used to encrypt and + decrypt the secrets. + properties: + secretRef: + description: |- + SecretRef is a reference to a secret that contains the private key. If this is set, Value must be nil. + Note that the value in the secret must be a 128-bit key encoded in hex. + properties: + key: + description: Key is the key in the secret that contains + the private key. + type: string + name: + description: Name is the name of the secret. + type: string + required: + - key + - name + type: object + value: + description: |- + Value is the private key. It must be a 128-bit key encoded in hex. If this is set, SecretRef must be nil. + When the feature gate RandomSecretStorePrivateKey is enabled and neither is set, the private key will be + generated randomly. + pattern: ^[0-9a-f]{32}$ + type: string + type: object + type: object standaloneMode: default: 0 description: |- diff --git a/config/risingwave-operator-test.yaml b/config/risingwave-operator-test.yaml index ee3e6934..9a313a7e 100644 --- a/config/risingwave-operator-test.yaml +++ b/config/risingwave-operator-test.yaml @@ -30527,6 +30527,38 @@ spec: - path type: object type: object + secretStore: + description: SecretStore is the configuration of the secret store. + properties: + privateKey: + description: PrivateKey is the private key used to encrypt and + decrypt the secrets. + properties: + secretRef: + description: |- + SecretRef is a reference to a secret that contains the private key. If this is set, Value must be nil. + Note that the value in the secret must be a 128-bit key encoded in hex. + properties: + key: + description: Key is the key in the secret that contains + the private key. + type: string + name: + description: Name is the name of the secret. + type: string + required: + - key + - name + type: object + value: + description: |- + Value is the private key. It must be a 128-bit key encoded in hex. If this is set, SecretRef must be nil. + When the feature gate RandomSecretStorePrivateKey is enabled and neither is set, the private key will be + generated randomly. + pattern: ^[0-9a-f]{32}$ + type: string + type: object + type: object standaloneMode: default: 0 description: |- diff --git a/config/risingwave-operator.yaml b/config/risingwave-operator.yaml index 7f3c4e66..573b0d69 100644 --- a/config/risingwave-operator.yaml +++ b/config/risingwave-operator.yaml @@ -30527,6 +30527,38 @@ spec: - path type: object type: object + secretStore: + description: SecretStore is the configuration of the secret store. + properties: + privateKey: + description: PrivateKey is the private key used to encrypt and + decrypt the secrets. + properties: + secretRef: + description: |- + SecretRef is a reference to a secret that contains the private key. If this is set, Value must be nil. + Note that the value in the secret must be a 128-bit key encoded in hex. + properties: + key: + description: Key is the key in the secret that contains + the private key. + type: string + name: + description: Name is the name of the secret. + type: string + required: + - key + - name + type: object + value: + description: |- + Value is the private key. It must be a 128-bit key encoded in hex. If this is set, SecretRef must be nil. + When the feature gate RandomSecretStorePrivateKey is enabled and neither is set, the private key will be + generated randomly. + pattern: ^[0-9a-f]{32}$ + type: string + type: object + type: object standaloneMode: default: 0 description: |- diff --git a/docs/general/api.md b/docs/general/api.md index 4e13d6b4..dcecae33 100644 --- a/docs/general/api.md +++ b/docs/general/api.md @@ -734,6 +734,19 @@ RisingWaveLicenseKey

LicenseKey to enable paid features of RisingWave.

+ + +secretStore
+ + +RisingWaveSecretStore + + + + +

SecretStore is the configuration of the secret store.

+ + @@ -4506,6 +4519,124 @@ k8s.io/apimachinery/pkg/types.UID +

RisingWaveSecretStore +

+

+(Appears on:RisingWaveSpec) +

+
+

RisingWaveSecretStore is the configuration of the secret store.

+
+ + + + + + + + + + + + + +
FieldDescription
+privateKey
+ + +RisingWaveSecretStorePrivateKey + + +		 +

PrivateKey is the private key used to encrypt and decrypt the secrets.

+
+

RisingWaveSecretStorePrivateKey +

+

+(Appears on:RisingWaveSecretStore) +

+
+

RisingWaveSecretStorePrivateKey is a private key that can be stored in a secret or directly in the resource.

+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+value
+ +string + +		 +(Optional) +

Value is the private key. It must be a 128-bit key encoded in hex. If this is set, SecretRef must be nil. +When the feature gate RandomSecretStorePrivateKey is enabled and neither is set, the private key will be +generated randomly.

+
+secretRef
+ + +RisingWaveSecretStorePrivateKeySecretReference + + +		 +(Optional) +

SecretRef is a reference to a secret that contains the private key. If this is set, Value must be nil. +Note that the value in the secret must be a 128-bit key encoded in hex.

+
+

RisingWaveSecretStorePrivateKeySecretReference +

+

+(Appears on:RisingWaveSecretStorePrivateKey) +

+
+

RisingWaveSecretStorePrivateKeySecretReference is a reference to a secret that contains a private key.

+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+name
+ +string + +		 +

Name is the name of the secret.

+
+key
+ +string + +		 +

Key is the key in the secret that contains the private key.

+

RisingWaveSpec

@@ -4736,6 +4867,19 @@ RisingWaveLicenseKey

LicenseKey to enable paid features of RisingWave.

+ + +secretStore
+ + +RisingWaveSecretStore + + + + +

SecretStore is the configuration of the secret store.

+ +

RisingWaveStandaloneComponent diff --git a/pkg/factory/envs/risingwave.go b/pkg/factory/envs/risingwave.go index 3939cf69..977d045a 100644 --- a/pkg/factory/envs/risingwave.go +++ b/pkg/factory/envs/risingwave.go @@ -23,36 +23,37 @@ const ( RustMinStack = "RUST_MIN_STACK" JavaOpts = "JAVA_OPTS" - RWListenAddr = "RW_LISTEN_ADDR" - RWAdvertiseAddr = "RW_ADVERTISE_ADDR" - RWDashboardHost = "RW_DASHBOARD_HOST" - RWPrometheusHost = "RW_PROMETHEUS_HOST" - RWEtcdEndpoints = "RW_ETCD_ENDPOINTS" - RWSQLEndpoint = "RW_SQL_ENDPOINT" - RWSQLDatabase = "RW_SQL_DATABASE" - RWEtcdAuth = "RW_ETCD_AUTH" - RWEtcdUsername = "RW_ETCD_USERNAME" - RWEtcdPassword = "RW_ETCD_PASSWORD" - RWSQLUsername = "RW_SQL_USERNAME" - RWSQLPassword = "RW_SQL_PASSWORD" - RWMySQLUsername = "RW_MYSQL_USERNAME" - RWMySQLPassword = "RW_MYSQL_PASSWORD" - RWPostgresUsername = "RW_POSTGRES_USERNAME" - RWPostgresPassword = "RW_POSTGRES_PASSWORD" - RWConfigPath = "RW_CONFIG_PATH" - RWStateStore = "RW_STATE_STORE" - RWDataDirectory = "RW_DATA_DIRECTORY" - RWWorkerThreads = "RW_WORKER_THREADS" - RWBackend = "RW_BACKEND" - RWMetaAddr = "RW_META_ADDR" - RWMetaAddrLegacy = "RW_META_ADDRESS" // Will deprecate soon. - RWPrometheusListenerAddr = "RW_PROMETHEUS_LISTENER_ADDR" - RWParallelism = "RW_PARALLELISM" - RWTotalMemoryBytes = "RW_TOTAL_MEMORY_BYTES" - RWSslCert = "RW_SSL_CERT" - RWSslKey = "RW_SSL_KEY" - RWLicenseKey = "RW_LICENSE_KEY" - RWLicenseKeyPath = "RW_LICENSE_KEY_PATH" + RWListenAddr = "RW_LISTEN_ADDR" + RWAdvertiseAddr = "RW_ADVERTISE_ADDR" + RWDashboardHost = "RW_DASHBOARD_HOST" + RWPrometheusHost = "RW_PROMETHEUS_HOST" + RWEtcdEndpoints = "RW_ETCD_ENDPOINTS" + RWSQLEndpoint = "RW_SQL_ENDPOINT" + RWSQLDatabase = "RW_SQL_DATABASE" + RWEtcdAuth = "RW_ETCD_AUTH" + RWEtcdUsername = "RW_ETCD_USERNAME" + RWEtcdPassword = "RW_ETCD_PASSWORD" + RWSQLUsername = "RW_SQL_USERNAME" + RWSQLPassword = "RW_SQL_PASSWORD" + RWMySQLUsername = "RW_MYSQL_USERNAME" + RWMySQLPassword = "RW_MYSQL_PASSWORD" + RWPostgresUsername = "RW_POSTGRES_USERNAME" + RWPostgresPassword = "RW_POSTGRES_PASSWORD" + RWConfigPath = "RW_CONFIG_PATH" + RWStateStore = "RW_STATE_STORE" + RWDataDirectory = "RW_DATA_DIRECTORY" + RWWorkerThreads = "RW_WORKER_THREADS" + RWBackend = "RW_BACKEND" + RWMetaAddr = "RW_META_ADDR" + RWMetaAddrLegacy = "RW_META_ADDRESS" // Will deprecate soon. + RWPrometheusListenerAddr = "RW_PROMETHEUS_LISTENER_ADDR" + RWParallelism = "RW_PARALLELISM" + RWTotalMemoryBytes = "RW_TOTAL_MEMORY_BYTES" + RWSslCert = "RW_SSL_CERT" + RWSslKey = "RW_SSL_KEY" + RWLicenseKey = "RW_LICENSE_KEY" + RWLicenseKeyPath = "RW_LICENSE_KEY_PATH" + RWSecretStorePrivateKeyHex = "RW_SECRET_STORE_PRIVATE_KEY_HEX" ) // MinIO. diff --git a/pkg/factory/risingwave_object_factory.go b/pkg/factory/risingwave_object_factory.go index 25e74e2a..f701b4c3 100644 --- a/pkg/factory/risingwave_object_factory.go +++ b/pkg/factory/risingwave_object_factory.go @@ -606,6 +606,34 @@ func (f *RisingWaveObjectFactory) envsForLicenseKey() []corev1.EnvVar { } } +func (f *RisingWaveObjectFactory) envsForSecretStore() []corev1.EnvVar { + secretStore := f.risingwave.Spec.SecretStore + if secretStore.PrivateKey.SecretRef != nil { + return []corev1.EnvVar{ + { + Name: envs.RWSecretStorePrivateKeyHex, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretStore.PrivateKey.SecretRef.Name, + }, + Key: secretStore.PrivateKey.SecretRef.Key, + }, + }, + }, + } + } + if secretStore.PrivateKey.Value != nil { + return []corev1.EnvVar{ + { + Name: envs.RWSecretStorePrivateKeyHex, + Value: *secretStore.PrivateKey.Value, + }, + } + } + return nil +} + func (f *RisingWaveObjectFactory) coreEnvsForMeta(image string) []corev1.EnvVar { metaStore := &f.risingwave.Spec.MetaStore version := utils.GetVersionFromImage(image) @@ -740,6 +768,7 @@ func (f *RisingWaveObjectFactory) coreEnvsForMeta(image string) []corev1.EnvVar } envVars = append(envVars, f.envsForLicenseKey()...) + envVars = append(envVars, f.envsForSecretStore()...) return envVars } diff --git a/pkg/factory/risingwave_object_factory_predicate_test.go b/pkg/factory/risingwave_object_factory_predicate_test.go index 55d9a522..4cde1b42 100644 --- a/pkg/factory/risingwave_object_factory_predicate_test.go +++ b/pkg/factory/risingwave_object_factory_predicate_test.go @@ -1166,3 +1166,38 @@ func licensePredicates() []predicate[*corev1.PodTemplateSpec, licenseTestCase] { }, } } + +func secretStorePredicates() []predicate[*corev1.PodTemplateSpec, secretStoreTestCase] { + return []predicate[*corev1.PodTemplateSpec, secretStoreTestCase]{ + { + Name: "envs-contain", + Fn: func(obj *corev1.PodTemplateSpec, testcase secretStoreTestCase) bool { + if len(testcase.envs) == 0 { + return true + } + if len(obj.Spec.Containers) == 0 { + return false + } + envs := obj.Spec.Containers[0].Env + // Contains all expected envs. + return listContainsByKey(envs, testcase.envs, func(t *corev1.EnvVar) string { return t.Name }, deepEqual[corev1.EnvVar]) + }, + }, + { + Name: "envs-not-contain", + Fn: func(obj *corev1.PodTemplateSpec, testcase secretStoreTestCase) bool { + if len(testcase.unexpectedEnvs) == 0 { + return true + } + if len(obj.Spec.Containers) == 0 { + return false + } + envs := obj.Spec.Containers[0].Env + // Contains none of unexpected envs. + return !lo.ContainsBy(envs, func(item corev1.EnvVar) bool { + return lo.Contains(testcase.unexpectedEnvs, item.Name) + }) + }, + }, + } +} diff --git a/pkg/factory/risingwave_object_factory_test.go b/pkg/factory/risingwave_object_factory_test.go index 5815ba9f..888c6404 100644 --- a/pkg/factory/risingwave_object_factory_test.go +++ b/pkg/factory/risingwave_object_factory_test.go @@ -588,3 +588,43 @@ func TestRisingWaveObjectFactory_DataDirectory(t *testing.T) { }) } } + +func TestRisingWaveObjectFactory_SecretStore(t *testing.T) { + predicates := secretStorePredicates() + + for name, tc := range secretStoreTestCases() { + t.Run(name, func(t *testing.T) { + factory := NewRisingWaveObjectFactory(newTestRisingwave(func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore = tc.secretStore + r.Spec.MetaStore.Memory = ptr.To(true) + r.Spec.StateStore.Memory = ptr.To(true) + r.Spec.Components.Meta.NodeGroups = []risingwavev1alpha1.RisingWaveNodeGroup{ + { + Name: "", + }, + } + }), testutils.Scheme, "") + + template := factory.NewMetaStatefulSet("").Spec.Template + composeAssertions(predicates, t).assertTest(&template, tc) + }) + } +} + +func TestRisingWaveObjectFactory_SecretStore_Standalone(t *testing.T) { + predicates := secretStorePredicates() + + for name, tc := range secretStoreTestCases() { + t.Run(name, func(t *testing.T) { + factory := NewRisingWaveObjectFactory(newTestRisingwave(func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore = tc.secretStore + r.Spec.MetaStore.Memory = ptr.To(true) + r.Spec.StateStore.Memory = ptr.To(true) + r.Spec.EnableStandaloneMode = ptr.To(true) + }), testutils.Scheme, "") + + template := factory.NewStandaloneStatefulSet().Spec.Template + composeAssertions(predicates, t).assertTest(&template, tc) + }) + } +} diff --git a/pkg/factory/risingwave_object_factory_testcases_test.go b/pkg/factory/risingwave_object_factory_testcases_test.go index 8cf7c963..1996409c 100644 --- a/pkg/factory/risingwave_object_factory_testcases_test.go +++ b/pkg/factory/risingwave_object_factory_testcases_test.go @@ -64,7 +64,8 @@ type testCaseType interface { metaAdvancedSTSTestCase | tlsTestcase | legacyLicenseTestCase | - licenseTestCase + licenseTestCase | + secretStoreTestCase } type kubeObject interface { @@ -4314,3 +4315,54 @@ func licenseTestCases() map[string]licenseTestCase { }, } } + +type secretStoreTestCase struct { + secretStore risingwavev1alpha1.RisingWaveSecretStore + envs []corev1.EnvVar + unexpectedEnvs []string +} + +func secretStoreTestCases() map[string]secretStoreTestCase { + return map[string]secretStoreTestCase{ + "no-secret-store": { + secretStore: risingwavev1alpha1.RisingWaveSecretStore{}, + unexpectedEnvs: []string{"RW_SECRET_STORE_PRIVATE_KEY_HEX"}, + }, + "with-secret-store": { + secretStore: risingwavev1alpha1.RisingWaveSecretStore{ + PrivateKey: risingwavev1alpha1.RisingWaveSecretStorePrivateKey{ + Value: ptr.To("0123456789abcdef0123456789abcdef"), + }, + }, + envs: []corev1.EnvVar{ + { + Name: "RW_SECRET_STORE_PRIVATE_KEY_HEX", + Value: "0123456789abcdef0123456789abcdef", + }, + }, + }, + "with-secret-store-secret-ref": { + secretStore: risingwavev1alpha1.RisingWaveSecretStore{ + PrivateKey: risingwavev1alpha1.RisingWaveSecretStorePrivateKey{ + SecretRef: &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "pk", + Key: "pk-key", + }, + }, + }, + envs: []corev1.EnvVar{ + { + Name: "RW_SECRET_STORE_PRIVATE_KEY_HEX", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "pk", + }, + Key: "pk-key", + }, + }, + }, + }, + }, + } +} diff --git a/pkg/features/feature_manager.go b/pkg/features/feature_manager.go index 03eb78e2..64d9962f 100644 --- a/pkg/features/feature_manager.go +++ b/pkg/features/feature_manager.go @@ -31,8 +31,9 @@ type FeatureName string // Valid feature names. const ( - EnableOpenKruiseFeature FeatureName = "EnableOpenKruise" - EnableForceUpdate FeatureName = "EnableForceUpdate" + EnableOpenKruiseFeature FeatureName = "EnableOpenKruise" + EnableForceUpdate FeatureName = "EnableForceUpdate" + RandomSecretStorePrivateKey FeatureName = "RandomSecretStorePrivateKey" ) // Valid feature stages. @@ -58,6 +59,13 @@ var ( DefaultEnable: true, Stage: Beta, }, + { + Name: RandomSecretStorePrivateKey, + Description: "This feature enables the random generation of a secret store private key if it is not set", + DefaultEnable: false, + Enabled: false, + Stage: Alpha, + }, } ) diff --git a/pkg/features/feature_manager_test.go b/pkg/features/feature_manager_test.go index 3004db3c..179e0fa2 100644 --- a/pkg/features/feature_manager_test.go +++ b/pkg/features/feature_manager_test.go @@ -434,3 +434,9 @@ func TestParseFromFeatureGateString(t *testing.T) { } } } + +func TestRandomSecretStorePrivateKeyIsDisabledByDefault(t *testing.T) { + if newRisingWaveFeatureManagerForTest("").IsFeatureEnabled(RandomSecretStorePrivateKey) { + t.Fatal("RandomSecretStorePrivateKey should be disabled by default") + } +} diff --git a/pkg/utils/rand.go b/pkg/utils/rand.go new file mode 100644 index 00000000..12d7f5f0 --- /dev/null +++ b/pkg/utils/rand.go @@ -0,0 +1,29 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "crypto/rand" + "encoding/hex" +) + +// RandomHex generates a random hex string of n bytes. +func RandomHex(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} diff --git a/pkg/webhook/risingwave_mutating_webhook.go b/pkg/webhook/risingwave_mutating_webhook.go index 27bf77ec..339eea1a 100644 --- a/pkg/webhook/risingwave_mutating_webhook.go +++ b/pkg/webhook/risingwave_mutating_webhook.go @@ -20,11 +20,15 @@ import ( "context" "strings" + "github.com/samber/lo" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/webhook" risingwavev1alpha1 "github.com/risingwavelabs/risingwave-operator/apis/risingwave/v1alpha1" + "github.com/risingwavelabs/risingwave-operator/pkg/features" "github.com/risingwavelabs/risingwave-operator/pkg/metrics" + "github.com/risingwavelabs/risingwave-operator/pkg/utils" ) // RisingWaveMutatingWebhook is the mutating webhook for RisingWaves. @@ -34,6 +38,15 @@ type RisingWaveMutatingWebhook struct{} func (m *RisingWaveMutatingWebhook) Default(ctx context.Context, obj runtime.Object) error { risingwave := obj.(*risingwavev1alpha1.RisingWave) risingwave.Spec.StateStore.DataDirectory = strings.TrimRight(strings.TrimSpace(risingwave.Spec.StateStore.DataDirectory), "/") + + if features.GetFeatureManager().IsFeatureEnabled(features.RandomSecretStorePrivateKey) { + // Generate a random private key if it is not set. + secretStorePrivateKey := &risingwave.Spec.SecretStore.PrivateKey + if secretStorePrivateKey.Value == nil && secretStorePrivateKey.SecretRef == nil { + secretStorePrivateKey.Value = ptr.To(lo.Must(utils.RandomHex(16))) + } + } + return nil } diff --git a/pkg/webhook/risingwave_mutating_webhook_test.go b/pkg/webhook/risingwave_mutating_webhook_test.go index e04414da..b50129ba 100644 --- a/pkg/webhook/risingwave_mutating_webhook_test.go +++ b/pkg/webhook/risingwave_mutating_webhook_test.go @@ -20,10 +20,13 @@ import ( "context" "testing" + "github.com/risingwavelabs/risingwave-operator/pkg/features" "github.com/risingwavelabs/risingwave-operator/pkg/testutils" ) func Test_RisingWaveMutatingWebhook_Default(t *testing.T) { + features.InitFeatureManager(features.SupportedFeatureList, "") + mutatingWebhook := NewRisingWaveMutatingWebhook() err := mutatingWebhook.Default(context.Background(), testutils.FakeRisingWave()) if err != nil { diff --git a/pkg/webhook/risingwave_validating_webhook.go b/pkg/webhook/risingwave_validating_webhook.go index f568530e..314d37b7 100644 --- a/pkg/webhook/risingwave_validating_webhook.go +++ b/pkg/webhook/risingwave_validating_webhook.go @@ -294,6 +294,14 @@ func (v *RisingWaveValidatingWebhook) validateMetaReplicas(obj *risingwavev1alph return fieldErrs } +func (v *RisingWaveValidatingWebhook) validateSecretStore(obj *risingwavev1alpha1.RisingWave) field.ErrorList { + fieldErrs := field.ErrorList{} + if obj.Spec.SecretStore.PrivateKey.Value != nil && obj.Spec.SecretStore.PrivateKey.SecretRef != nil { + fieldErrs = append(fieldErrs, field.Forbidden(field.NewPath("spec", "secretStore", "privateKey"), "both value and secretRef are set")) + } + return fieldErrs +} + func (v *RisingWaveValidatingWebhook) validateCreate(ctx context.Context, obj *risingwavev1alpha1.RisingWave) error { gvk := obj.GroupVersionKind() @@ -334,6 +342,9 @@ func (v *RisingWaveValidatingWebhook) validateCreate(ctx context.Context, obj *r // Validate the meta replicas. fieldErrs = append(fieldErrs, v.validateMetaReplicas(obj)...) + // Validate the secret store. + fieldErrs = append(fieldErrs, v.validateSecretStore(obj)...) + if len(fieldErrs) > 0 { return apierrors.NewInvalid(gvk.GroupKind(), obj.Name, fieldErrs) } @@ -368,6 +379,24 @@ func pathForGroupReplicas(obj *risingwavev1alpha1.RisingWave, component, group s return field.NewPath("spec", "components", component, "nodeGroups").Index(index).Child("replicas") } +func (v *RisingWaveValidatingWebhook) isSecretStoreChangeAllowed(oldObj, newObj *risingwavev1alpha1.RisingWave) bool { + oldStore, newStore := oldObj.Spec.SecretStore, newObj.Spec.SecretStore + + isPrivateKeySet := func(store *risingwavev1alpha1.RisingWaveSecretStore) bool { + return store.PrivateKey.Value != nil || store.PrivateKey.SecretRef != nil + } + + // Not set to set is allowed. + if !isPrivateKeySet(&oldStore) { + return true + } + + // Changes on the private key are not allowed. + oldPk, newPk := oldStore.PrivateKey, newStore.PrivateKey + + return equality.Semantic.DeepEqual(oldPk, newPk) +} + func (v *RisingWaveValidatingWebhook) validateUpdate(ctx context.Context, oldObj, newObj *risingwavev1alpha1.RisingWave) error { gvk := oldObj.GroupVersionKind() @@ -388,6 +417,14 @@ func (v *RisingWaveValidatingWebhook) validateUpdate(ctx context.Context, oldObj ) } + if !v.isSecretStoreChangeAllowed(oldObj, newObj) { + return apierrors.NewForbidden( + schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, + oldObj.Name, + field.Forbidden(field.NewPath("spec", "secretStore"), "secret store must be kept consistent"), + ) + } + fieldErrs := field.ErrorList{} // Validate the locks from scale views. diff --git a/pkg/webhook/risingwave_validating_webhook_test.go b/pkg/webhook/risingwave_validating_webhook_test.go index 49a06887..9034dd87 100644 --- a/pkg/webhook/risingwave_validating_webhook_test.go +++ b/pkg/webhook/risingwave_validating_webhook_test.go @@ -22,6 +22,7 @@ import ( "testing" kruisepubs "github.com/openkruise/kruise-api/apps/pub" + "github.com/samber/lo" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" @@ -32,6 +33,7 @@ import ( risingwavev1alpha1 "github.com/risingwavelabs/risingwave-operator/apis/risingwave/v1alpha1" "github.com/risingwavelabs/risingwave-operator/pkg/consts" "github.com/risingwavelabs/risingwave-operator/pkg/testutils" + "github.com/risingwavelabs/risingwave-operator/pkg/utils" ) func Test_RisingWaveValidatingWebhook_ValidateDelete(t *testing.T) { @@ -787,6 +789,43 @@ func Test_RisingWaveValidatingWebhook_ValidateCreate(t *testing.T) { }, pass: true, }, + "empty-secret-store-private-key": { + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey = risingwavev1alpha1.RisingWaveSecretStorePrivateKey{} + }, + pass: true, + }, + "secret-store-private-key-value-set": { + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey = risingwavev1alpha1.RisingWaveSecretStorePrivateKey{ + Value: ptr.To("123"), + } + }, + pass: true, + }, + "secret-store-private-key-ref-set": { + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey = risingwavev1alpha1.RisingWaveSecretStorePrivateKey{ + SecretRef: &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "test", + Key: "test", + }, + } + }, + pass: true, + }, + "secret-store-private-key-both-set": { + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey = risingwavev1alpha1.RisingWaveSecretStorePrivateKey{ + Value: ptr.To("123"), + SecretRef: &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "test", + Key: "test", + }, + } + }, + pass: false, + }, } for name, tc := range testcases { @@ -809,6 +848,7 @@ func Test_RisingWaveValidatingWebhook_ValidateCreate(t *testing.T) { func Test_RisingWaveValidatingWebhook_ValidateUpdate(t *testing.T) { testcases := map[string]struct { + init func(r *risingwavev1alpha1.RisingWave) patch func(r *risingwavev1alpha1.RisingWave) pass bool openKruiseAvailable bool @@ -875,6 +915,103 @@ func Test_RisingWaveValidatingWebhook_ValidateUpdate(t *testing.T) { }, pass: false, }, + "secret-store-nil-unchanged-success": { // nil secret store + patch: func(r *risingwavev1alpha1.RisingWave) {}, + pass: true, + }, + "secret-store-changed-from-nil-success-0": { + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.Value = ptr.To(lo.Must(utils.RandomHex(32))) + }, + pass: true, + }, + "secret-store-changed-from-nil-success-1": { + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.SecretRef = &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "test", + Key: "test", + } + }, + pass: true, + }, + "secret-store-changed-from-non-nil-to-nil-fail-0": { + init: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.Value = ptr.To(lo.Must(utils.RandomHex(32))) + }, + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore = risingwavev1alpha1.RisingWaveSecretStore{} + }, + pass: false, + }, + "secret-store-changed-from-non-nil-to-nil-fail-1": { + init: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.SecretRef = &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "test", + Key: "test", + } + }, + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore = risingwavev1alpha1.RisingWaveSecretStore{} + }, + pass: false, + }, + "secret-store-unchanged-from-value-to-value-success": { + init: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.Value = ptr.To("123") + }, + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.Value = ptr.To("123") + }, + pass: true, + }, + "secret-store-unchanged-from-ref-to-ref-success": { + init: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.SecretRef = &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "test", + Key: "test", + } + }, + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.SecretRef = &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "test", + Key: "test", + } + }, + pass: true, + }, + "secret-store-changed-from-value-to-value-fail": { + init: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.Value = ptr.To("123") + }, + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.Value = ptr.To("456") + }, + pass: false, + }, + "secret-store-changed-from-value-to-secret-fail": { + init: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.Value = ptr.To("123") + }, + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.SecretRef = &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "test", + Key: "test", + } + }, + pass: false, + }, + "secret-store-changed-from-secret-to-value-fail": { + init: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.SecretRef = &risingwavev1alpha1.RisingWaveSecretStorePrivateKeySecretReference{ + Name: "test", + Key: "test", + } + }, + patch: func(r *risingwavev1alpha1.RisingWave) { + r.Spec.SecretStore.PrivateKey.Value = ptr.To("123") + }, + pass: false, + }, } for name, tc := range testcases { @@ -883,6 +1020,10 @@ func Test_RisingWaveValidatingWebhook_ValidateUpdate(t *testing.T) { // We want to create two copies, so we can compare the old state and new state // when transitioning from openkruise enabled to disabled with operator disabled. risingwave := testutils.FakeRisingWave() + if tc.init != nil { + tc.init(risingwave) + } + oldObj := risingwave.DeepCopy() if tc.oldObjMutation != nil { tc.oldObjMutation(oldObj)