Make Reaper's storage configurable + adjust its deployment accordingly

CHANGELOG/CHANGELOG-1.18.md

@@ -23,3 +23,5 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 * [ENHANCEMENT] [#1274](https://github.com/k8ssandra/k8ssandra-operator/issues/1274) On upgrade, do not modify the CassandraDatacenter object unless instructed with an annotation `k8ssandra.io/autoupdate-spec` with value `once` or `always`
 * [BUGFIX] [#1222](https://github.com/k8ssandra/k8ssandra-operator/issues/1222) Consider DC-level config when validating numToken updates in webhook
 * [BUGFIX] [#1366](https://github.com/k8ssandra/k8ssandra-operator/issues/1366) Reaper deployment can't be created on OpenShift due to missing RBAC rule
+* [CHANGE] Update cassandra-medusa to 0.22.0
+* [FEATURE] [#1275](https://github.com/k8ssandra/k8ssandra-operator/issues/1275) Allow configuring Reaper to use a memory storage backend

apis/k8ssandra/v1alpha1/k8ssandracluster_webhook.go

@@ -19,6 +19,7 @@ package v1alpha1
 import (
 	"fmt"
 	"github.com/Masterminds/semver/v3"
+	reaperapi "github.com/k8ssandra/k8ssandra-operator/apis/reaper/v1alpha1"
 	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,13 +35,17 @@ import (
 )
 
 var (
-	clientCache        *clientcache.ClientCache
-	ErrNumTokens       = fmt.Errorf("num_tokens value can't be changed")
-	ErrReaperKeyspace  = fmt.Errorf("reaper keyspace can not be changed")
-	ErrNoStorageConfig = fmt.Errorf("storageConfig must be defined at cluster level or dc level")
-	ErrNoResourcesSet  = fmt.Errorf("softPodAntiAffinity requires Resources to be set")
-	ErrClusterName     = fmt.Errorf("cluster name can not be changed")
-	ErrNoStoragePrefix = fmt.Errorf("medusa storage prefix must be set when a medusaConfigurationRef is used")
+	clientCache                 *clientcache.ClientCache
+	ErrNumTokens                = fmt.Errorf("num_tokens value can't be changed")
+	ErrReaperKeyspace           = fmt.Errorf("reaper keyspace can not be changed")
+	ErrNoStorageConfig          = fmt.Errorf("storageConfig must be defined at cluster level or dc level")
+	ErrNoResourcesSet           = fmt.Errorf("softPodAntiAffinity requires Resources to be set")
+	ErrClusterName              = fmt.Errorf("cluster name can not be changed")
+	ErrNoStoragePrefix          = fmt.Errorf("medusa storage prefix must be set when a medusaConfigurationRef is used")
+	ErrNoReaperStorageConfig    = fmt.Errorf("reaper StorageConfig not set")
+	ErrNoReaperAccessMode       = fmt.Errorf("reaper StorageConfig.AccessModes not set")
+	ErrNoReaperResourceRequests = fmt.Errorf("reaper StorageConfig.Resources.Requests not set")
+	ErrNoReaperStorageRequest   = fmt.Errorf("reaper StorageConfig.Resources.Requests.Storage not set")
 )
 
 // log is for logging in this package.
@@ -113,6 +118,10 @@ func (r *K8ssandraCluster) validateK8ssandraCluster() error {
 		return err
 	}
 
+	if err := r.validateReaper(); err != nil {
+		return err
+	}
+
 	if err := r.validateStatefulsetNameSize(); err != nil {
 		return err
 	}
@@ -280,3 +289,26 @@ func (r *K8ssandraCluster) ValidateMedusa() error {
 
 	return nil
 }
+
+func (r *K8ssandraCluster) validateReaper() error {
+	if r.Spec.Reaper == nil {
+		return nil
+	}
+	if r.Spec.Reaper.StorageType != reaperapi.StorageTypeLocal {
+		return nil
+	}
+	if r.Spec.Reaper.StorageConfig == nil {
+		return ErrNoReaperStorageConfig
+	}
+	// not checking StorageClassName because Kubernetes will use a default one if it's not set
+	if r.Spec.Reaper.StorageConfig.AccessModes == nil {
+		return ErrNoReaperAccessMode
+	}
+	if r.Spec.Reaper.StorageConfig.Resources.Requests == nil {
+		return ErrNoReaperResourceRequests
+	}
+	if r.Spec.Reaper.StorageConfig.Resources.Requests.Storage().IsZero() {
+		return ErrNoReaperStorageRequest
+	}
+	return nil
+}

apis/k8ssandra/v1alpha1/k8ssandracluster_webhook_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"net"
 	"path/filepath"
 	"testing"
@@ -53,6 +54,23 @@ var testEnv *envtest.Environment
 var ctx context.Context
 var cancel context.CancelFunc
 
+var minimalInMemoryReaperStorageConfig = &corev1.PersistentVolumeClaimSpec{
+	StorageClassName: func() *string { s := "test"; return &s }(),
+	AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+	Resources: corev1.VolumeResourceRequirements{
+		Requests: corev1.ResourceList{
+			corev1.ResourceStorage: resource.MustParse("1Gi"),
+		},
+	},
+}
+
+var minimalInMemoryReaperConfig = &reaperapi.ReaperClusterTemplate{
+	ReaperTemplate: reaperapi.ReaperTemplate{
+		StorageType:   reaperapi.StorageTypeLocal,
+		StorageConfig: minimalInMemoryReaperStorageConfig,
+	},
+}
+
 func TestWebhook(t *testing.T) {
 	required := require.New(t)
 	ctx, cancel = context.WithCancel(context.TODO())
@@ -160,6 +178,7 @@ func TestWebhook(t *testing.T) {
 	t.Run("InvalidDcName", testInvalidDcName)
 	t.Run("MedusaConfigNonLocalNamespace", testMedusaNonLocalNamespace)
 	t.Run("AutomatedUpdateAnnotation", testAutomatedUpdateAnnotation)
+	t.Run("ReaperStorage", testReaperStorage)
 }
 
 func testContextValidation(t *testing.T) {
@@ -496,6 +515,36 @@ func testMedusaNonLocalNamespace(t *testing.T) {
 	required.Contains(err.Error(), "Medusa config must be namespace local")
 }
 
+func testReaperStorage(t *testing.T) {
+	required := require.New(t)
+
+	reaperWithNoStorageConfig := createMinimalClusterObj("reaper-no-storage-config", "ns")
+	reaperWithNoStorageConfig.Spec.Reaper = &reaperapi.ReaperClusterTemplate{
+		ReaperTemplate: reaperapi.ReaperTemplate{
+			StorageType: reaperapi.StorageTypeLocal,
+		},
+	}
+	err := reaperWithNoStorageConfig.validateK8ssandraCluster()
+	required.Error(err)
+
+	reaperWithDefaultConfig := createClusterObjWithCassandraConfig("reaper-default-storage-config", "ns")
+	reaperWithDefaultConfig.Spec.Reaper = minimalInMemoryReaperConfig.DeepCopy()
+	err = reaperWithDefaultConfig.validateK8ssandraCluster()
+	required.NoError(err)
+
+	reaperWithoutAccessMode := createClusterObjWithCassandraConfig("reaper-no-access-mode", "ns")
+	reaperWithoutAccessMode.Spec.Reaper = minimalInMemoryReaperConfig.DeepCopy()
+	reaperWithoutAccessMode.Spec.Reaper.StorageConfig.AccessModes = nil
+	err = reaperWithoutAccessMode.validateK8ssandraCluster()
+	required.Error(err)
+
+	reaperWithoutStorageSize := createClusterObjWithCassandraConfig("reaper-no-storage-size", "ns")
+	reaperWithoutStorageSize.Spec.Reaper = minimalInMemoryReaperConfig.DeepCopy()
+	reaperWithoutStorageSize.Spec.Reaper.StorageConfig.Resources.Requests = corev1.ResourceList{}
+	err = reaperWithoutStorageSize.validateK8ssandraCluster()
+	required.Error(err)
+}
+
 // TestValidateUpdateNumTokens is a unit test for numTokens updates.
 func TestValidateUpdateNumTokens(t *testing.T) {
 	type config struct {

apis/reaper/v1alpha1/reaper_types.go

@@ -30,12 +30,29 @@ import (
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
 const (
-	ReaperLabel     = "k8ssandra.io/reaper"
-	DefaultKeyspace = "reaper_db"
+	DeploymentModeSingle = "SINGLE"
+	DeploymentModePerDc  = "PER_DC"
+	ReaperLabel          = "k8ssandra.io/reaper"
+	DefaultKeyspace      = "reaper_db"
+	StorageTypeCassandra = "cassandra"
+	StorageTypeLocal     = "local"
 )
 
 type ReaperTemplate struct {
 
+	// The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store
+	// its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster).
+	// The "local" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters.
+	// +kubebuilder:validation:Enum=cassandra;local
+	// +kubebuilder:default="cassandra"
+	// +optional
+	StorageType string `json:"storageType,omitempty"`
+
+	// If StorageType is "local", Reaper will need a Persistent Volume to persist its data. This field allows
+	// configuring that Persistent Volume.
+	// +optional
+	StorageConfig *corev1.PersistentVolumeClaimSpec `json:"storageConfig,omitempty"`
+
 	// The keyspace to use to store Reaper's state. Will default to "reaper_db" if unspecified. Will be created if it
 	// does not exist, and if this Reaper resource is managed by K8ssandra.
 	// +kubebuilder:default="reaper_db"
@@ -222,6 +239,20 @@ type ReaperClusterTemplate struct {
 	DeploymentMode string `json:"deploymentMode,omitempty"`
 }
 
+// EnsureDeploymentMode ensures that a deployment mode is SINGLE if we use the local storage type. This is to prevent
+// several instances of Reapers with local storage that would interfere with each other.
+func (t *ReaperClusterTemplate) EnsureDeploymentMode() bool {
+	if t != nil {
+		if t.StorageType == StorageTypeLocal {
+			if t.DeploymentMode != DeploymentModeSingle {
+				t.DeploymentMode = DeploymentModeSingle
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // CassandraDatacenterRef references the target Cassandra DC that Reaper should manage.
 // TODO this object could be used by Stargate too; which currently cannot locate DCs outside of its own namespace.
 type CassandraDatacenterRef struct {

apis/reaper/v1alpha1/reaper_types_test.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestEnsureDeploymentMode(t *testing.T) {
+	rct := &ReaperClusterTemplate{
+		ReaperTemplate: ReaperTemplate{
+			StorageType: StorageTypeLocal,
+		},
+		DeploymentMode: DeploymentModePerDc,
+	}
+	changed := rct.EnsureDeploymentMode()
+	assert.True(t, changed)
+	assert.Equal(t, DeploymentModeSingle, rct.DeploymentMode)
+
+	rct = &ReaperClusterTemplate{
+		ReaperTemplate: ReaperTemplate{
+			StorageType: StorageTypeCassandra,
+		},
+		DeploymentMode: DeploymentModePerDc,
+	}
+	changed = rct.EnsureDeploymentMode()
+	assert.False(t, changed)
+	assert.Equal(t, DeploymentModePerDc, rct.DeploymentMode)
+}