Support redunce volume size (#552)

* Support redunce volume size Signed-off-by: d-kuro <kurosawa7620@gmail.com> * In the upgrade tests, the tests for PVC are skipped. Signed-off-by: d-kuro <kurosawa7620@gmail.com> * Rename and document PVC-related function names in tests Signed-off-by: d-kuro <kurosawa7620@gmail.com> * Add docs Signed-off-by: d-kuro <kurosawa7620@gmail.com> * Add test for inserting data into MySQL Signed-off-by: d-kuro <kurosawa7620@gmail.com> * Fix pvc reduce tests Signed-off-by: d-kuro <kurosawa7620@gmail.com> --------- Signed-off-by: d-kuro <kurosawa7620@gmail.com>
cybozu-go · Sep 6, 2023 · 00cce35 · 00cce35
1 parent 720adf8
commit 00cce35
Show file tree

Hide file tree

Showing 9 changed files with 337 additions and 113 deletions.
diff --git a/api/v1beta2/mysqlcluster_types.go b/api/v1beta2/mysqlcluster_types.go
@@ -271,9 +271,9 @@ func (s MySQLClusterSpec) validateUpdate(ctx context.Context, apiReader client.R
 
 			switch cmp := newSize.Cmp(oldSize); {
 			case cmp == -1:
-				p := p.Child("volumeClaimTemplates").Index(i).
-					Child("spec").Child("resources").Child("requests").Key("storage")
-				allErrs = append(allErrs, field.Forbidden(p, "storage size cannot be reduced"))
+				// noop
+				// Allow users to reduce the volume size by operating.
+				// ref: docs/designdoc/support_reduce_volume_size.md
 			case cmp == 1:
 				volumeExpansionTargetIndices = append(volumeExpansionTargetIndices, i)
 			case cmp == 0:

diff --git a/api/v1beta2/mysqlcluster_webhook_test.go b/api/v1beta2/mysqlcluster_webhook_test.go
@@ -498,25 +498,6 @@ var _ = Describe("MySQLCluster Webhook", func() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
-	It("should deny reduced storage size", func() {
-		r := makeMySQLCluster()
-		err := k8sClient.Create(ctx, r)
-		Expect(err).NotTo(HaveOccurred())
-
-		r.Spec.VolumeClaimTemplates[0].Spec = mocov1beta2.PersistentVolumeClaimSpecApplyConfiguration(
-			*corev1ac.PersistentVolumeClaimSpec().
-				WithStorageClassName("default").
-				WithResources(corev1ac.ResourceRequirements().
-					WithRequests(corev1.ResourceList{
-						corev1.ResourceStorage: resource.MustParse("1Mi"),
-					}),
-				),
-		)
-
-		err = k8sClient.Update(ctx, r)
-		Expect(err).To(HaveOccurred())
-	})
-
 	It("should deny storage size expansion for not support volume expansion storage class", func() {
 		r := makeMySQLCluster()
 		r.Spec.VolumeClaimTemplates = make([]mocov1beta2.PersistentVolumeClaim, 2)

diff --git a/controllers/pvc.go b/controllers/pvc.go
@@ -131,6 +131,9 @@ func (r *MySQLClusterReconciler) resizePVCs(ctx context.Context, cluster *mocov1
 		case i == 0: // volume size is equal
 			continue
 		case i == 1: // current volume size is greater than new size
+			// The size of the Persistent Volume Claims (PVC) cannot be reduced.
+			// Although MOCO permits the reduction of PVC, an automatic resize is not performed.
+			// An error arises if a PVC involving reduction is passed, as it's unexpected.
 			return resizedPVC, fmt.Errorf("failed to resize pvc %q, want size: %s, deployed size: %s: %w", pvc.Name, newSize.String(), pvc.Spec.Resources.Requests.Storage().String(), ErrReduceVolumeSize)
 		case i == -1: // current volume size is smaller than new size
 			pvc.Spec.Resources.Requests[corev1.ResourceStorage] = *newSize
@@ -174,7 +177,12 @@ func (*MySQLClusterReconciler) needResizePVC(cluster *mocov1beta2.MySQLCluster,
 		case i == 0: // volume size is equal
 			continue
 		case i == 1: // volume size is greater
-			return nil, false, fmt.Errorf("failed to resize pvc %q, want size: %s, deployed size: %s: %w", pvc.Name, wantSize, deployedSize, ErrReduceVolumeSize)
+			// Due to the lack of support for volume size reduction, resizing will not be executed if it implies a smaller size.
+			// It's important to highlight that this does not induce an error.
+			// Instead, the recreation of the StatefulSet will be managed in the reconcileV1StatefulSet() operation, which follows this one.
+			// Hence, the execution flow remains uninterrupted.
+			// ref: docs/designdoc/support_reduce_volume_size.md
+			continue
 		case i == -1: // volume size is smaller
 			resizeTarget[pvc.Name] = pvcSet[pvc.Name]
 			continue

diff --git a/controllers/pvc_test.go b/controllers/pvc_test.go
@@ -159,7 +159,6 @@ func TestNeedResizePVC(t *testing.T) {
 			cluster:    newMySQLClusterWithVolumeSize(resource.MustParse("1Gi")),
 			sts:        newStatefulSetWithVolumeSize(resource.MustParse("2Gi")),
 			wantResize: false,
-			wantError:  ErrReduceVolumeSize,
 		},
 		{
 			name:    "StatefulSet has more PVCs",
@@ -201,14 +200,55 @@ func TestNeedResizePVC(t *testing.T) {
 			sts:        newStatefulSetWithVolumeSize(resource.MustParse("1Gi")),
 			wantResize: false,
 		},
+		{
+			name: "Mix expansion and reduction",
+			cluster: func() *mocov1beta2.MySQLCluster {
+				cluster := newMySQLClusterWithVolumeSize(resource.MustParse("2Gi"))
+				pvc := mocov1beta2.PersistentVolumeClaim{
+					ObjectMeta: mocov1beta2.ObjectMeta{Name: "new-data"},
+					Spec: mocov1beta2.PersistentVolumeClaimSpecApplyConfiguration(*corev1ac.PersistentVolumeClaimSpec().
+						WithStorageClassName("default").WithResources(corev1ac.ResourceRequirements().
+						WithRequests(corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}),
+					)),
+				}
+				cluster.Spec.VolumeClaimTemplates = append(cluster.Spec.VolumeClaimTemplates, pvc)
+				return cluster
+			}(),
+			sts: func() *appsv1.StatefulSet {
+				sts := newStatefulSetWithVolumeSize(resource.MustParse("1Gi"))
+				pvc := corev1.PersistentVolumeClaim{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "new-data",
+					},
+					Spec: corev1.PersistentVolumeClaimSpec{
+						StorageClassName: pointer.String("default"),
+						Resources: corev1.ResourceRequirements{
+							Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("2Gi")},
+						},
+					},
+				}
+
+				sts.Spec.VolumeClaimTemplates = append(sts.Spec.VolumeClaimTemplates, pvc)
+
+				return sts
+			}(),
+			wantResizeTarget: func() map[string]corev1.PersistentVolumeClaim {
+				sts := newStatefulSetWithVolumeSize(resource.MustParse("1Gi"))
+				pvc := sts.Spec.VolumeClaimTemplates[0]
+				m := make(map[string]corev1.PersistentVolumeClaim)
+				m[pvc.Name] = pvc
+				return m
+			}(),
+			wantResize: true,
+		},
 	}
 
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			r := &MySQLClusterReconciler{}
 			resizeTarget, resize, err := r.needResizePVC(tt.cluster, tt.sts)
-			if err != nil {
+			if tt.wantError != nil {
 				if !errors.Is(err, tt.wantError) {
 					t.Fatalf("want error %v, got %v", tt.wantError, err)
 				}
@@ -224,7 +264,7 @@ func TestNeedResizePVC(t *testing.T) {
 
 			for key, value := range tt.wantResizeTarget {
 				if diff := cmp.Diff(value, resizeTarget[key]); len(diff) != 0 {
-					t.Fatalf("want resize target %v, got %v", value, resizeTarget[key])
+					t.Fatalf("unexpected resize target: %s", diff)
 				}
 			}
 		})

diff --git a/docs/change-pvc-template.md b/docs/change-pvc-template.md
@@ -62,6 +62,99 @@ so users can notice anomalies by monitoring this metrics.
 
 See the [metrics documentation](./metrics.md) for more details.
 
+## Volume reduction
+
+MOCO supports PVC reduction, but unlike PVC expansion, the user must perform the operation manually.
+
+The steps are as follows:
+
+1. The user modifies the `.spec.volumeClaimTemplates` of the MySQLCluster and sets a smaller volume size.
+2. MOCO updates the `.spec.volumeClaimTemplates` of the StatefulSet. This does not propagate to existing Pods, PVCs, or PVs.
+3. The user manually deletes the MySQL Pod & PVC.
+4. Wait for the Pod & PVC to be recreated by the statefulset-controller, and for MOCO to clone the data.
+5. Once the cluster becomes Healthy, the user deletes the next Pod and PVC.
+6. It is completed when all Pods and PVCs are recreated.
+
+### 1. The user modifies the `.spec.volumeClaimTemplates` of the MySQLCluster and sets a smaller volume size
+
+For example, the user modifies the `.spec.volumeClaimTemplates` of the MySQLCluster as follows:
+
+```diff
+  apiVersion: moco.cybozu.com/v1beta2
+  kind: MySQLCluster
+  metadata:
+    namespace: default
+    name: test
+  spec:
+    replicas: 3
+    podTemplate:
+      spec:
+        containers:
+        - name: mysqld
+          image: quay.io/cybozu/mysql:8.0.30
+    volumeClaimTemplates:
+    - metadata:
+        name: mysql-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+-           storage: 1Gi
++           storage: 500Mi
+```
+
+### 2. MOCO updates the `.spec.volumeClaimTemplates` of the StatefulSet. This does not propagate to existing Pods, PVCs, or PVs
+
+The moco-controller will update the `.spec.volumeClaimTemplates` of the StatefulSet.
+The actual modification of the StatefulSet's `.spec.volumeClaimTemplates` is not allowed,
+so this change is achieved by recreating the StatefulSet.
+At this time, only the recreation of StatefulSet is performed, without deleting the Pods and PVCs.
+
+### 3. The user manually deletes the MySQL Pod & PVC
+
+The user manually deletes the PVC and Pod.
+Use the following command to delete them:
+
+```console
+$ kubectl delete --wait=false pvc <pvc-name>
+$ kubectl delete --grace-period=1 <pod-name>
+```
+
+### 4. Wait for the Pod & PVC to be recreated by the statefulset-controller, and for MOCO to clone the data
+
+The statefulset-controller recreates Pods and PVCs, creating a new PVC with a reduced size.
+Once the MOCO successfully starts a Pod, it begins cloning the data.
+
+```console
+$ kubectl get mysqlcluster,po,pvc
+NAME                                AVAILABLE   HEALTHY   PRIMARY   SYNCED REPLICAS   ERRANT REPLICAS   LAST BACKUP
+mysqlcluster.moco.cybozu.com/test   True        False     0         2                                   <no value>
+
+NAME              READY   STATUS     RESTARTS   AGE
+pod/moco-test-0   3/3     Running    0          2m14s
+pod/moco-test-1   3/3     Running    0          114s
+pod/moco-test-2   0/3     Init:1/2   0          7s
+
+NAME                                           STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+persistentvolumeclaim/mysql-data-moco-test-0   Bound    pvc-03c73525-0d6d-49de-b68a-f8af4c4c7faa   1Gi        RWO            standard       2m14s
+persistentvolumeclaim/mysql-data-moco-test-1   Bound    pvc-73c26baa-3432-4c85-b5b6-875ffd2456d9   1Gi        RWO            standard       114s
+persistentvolumeclaim/mysql-data-moco-test-2   Bound    pvc-779b5b3c-3efc-4048-a549-a4bd2d74ed4e   500Mi      RWO            standard       7s
+```
+
+### 5. Once the cluster becomes Healthy, the user deletes the next Pod and PVC
+
+The user waits until the MySQLCluster state becomes Healthy, and then deletes the next Pod and PVC.
+
+```console
+$ kubectl get mysqlcluster
+NAME                                AVAILABLE   HEALTHY   PRIMARY   SYNCED REPLICAS   ERRANT REPLICAS   LAST BACKUP
+mysqlcluster.moco.cybozu.com/test   True        True      1         3                                   <no value>
+```
+
+### 6. It is completed when all Pods and PVCs are recreated
+
+Repeat steps 3 to 5 until all Pods and PVCs are recreated.
+
 ## References
 
 * [Design document](./designdoc/support_apply_pvc_template_changes.md)