K8SPXC-1341: Resize PVCs on user request (#1588)

* K8SPXC-1341: Resize PVCs on user request

* Apply suggestions from code review

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

e2e-tests/pvc-resize/compare/select-1.sql

@@ -0,0 +1 @@
+100500

e2e-tests/pvc-resize/run

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -o errexit
+
+test_dir=$(realpath $(dirname $0))
+. ${test_dir}/../functions
+
+set_debug
+
+create_infra ${namespace}
+
+desc 'create first PXC cluster'
+cluster="some-name"
+spinup_pxc "${cluster}" "$conf_dir/$cluster.yml" "3" "10" "${conf_dir}/secrets.yml"
+
+kubectl_bin patch pxc "$cluster" --type=merge --patch '{
+    "spec": { "pxc": { "volumeSpec": { "persistentVolumeClaim": { "resources": { "requests": { "storage": "4Gi" } } } } } }
+}'
+wait_cluster_consistency "$cluster" 3 2
+
+for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o name); do
+	retry=0
+	until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "4Gi" ]]; do
+		if [[ $retry -ge 60 ]]; then
+			echo "PVC ${pvc} was not resized, max retries exceeded"
+			exit 1
+		fi
+
+		echo "Waiting for PVC ${pvc} to be resized"
+		sleep 5
+
+		retry=$((retry + 1))
+	done
+	echo "PVC ${pvc} was resized"
+done
+
+destroy "${namespace}"
+desc "test passed"

e2e-tests/run-pr.csv

@@ -35,3 +35,5 @@ upgrade-proxysql,8.0
 users,5.7
 users,8.0
 validation-hook,8.0
+pvc-resize,5.7
+pvc-resize,8.0

pkg/controller/pxc/controller.go

@@ -294,6 +294,10 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(ctx context.Context, request r
 			log.Info("failed to ensure version, running with default", "error", err)
 		}
 	}
+	err = r.reconcilePersistentVolumes(ctx, o)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrap(err, "reconcile persistent volumes")
+	}
 
 	err = r.deploy(ctx, o)
 	if err != nil {

pkg/controller/pxc/volumes.go

@@ -0,0 +1,95 @@
+package pxc
+
+import (
+	"context"
+	"strings"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1"
+	"github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app/statefulset"
+)
+
+func (r *ReconcilePerconaXtraDBCluster) reconcilePersistentVolumes(ctx context.Context, cr *api.PerconaXtraDBCluster) error {
+	log := logf.FromContext(ctx)
+
+	pxcSet := statefulset.NewNode(cr)
+	sts := pxcSet.StatefulSet()
+
+	err := r.client.Get(ctx, client.ObjectKeyFromObject(sts), sts)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "get statefulset/%s", sts.Name)
+	}
+
+	if cr.Spec.PXC.VolumeSpec.PersistentVolumeClaim == nil {
+		return nil
+	}
+
+	var volumeTemplate corev1.PersistentVolumeClaim
+	for _, vct := range sts.Spec.VolumeClaimTemplates {
+		if vct.Name == "datadir" {
+			volumeTemplate = vct
+		}
+	}
+
+	requested := cr.Spec.PXC.VolumeSpec.PersistentVolumeClaim.Resources.Requests[corev1.ResourceStorage]
+	actual := volumeTemplate.Spec.Resources.Requests[corev1.ResourceStorage]
+
+	if requested.Cmp(actual) < 0 {
+		return errors.Wrap(err, "requested storage is less than actual")
+	}
+
+	if requested.Cmp(actual) == 0 {
+		return nil
+	}
+
+	// Check storage class for allowVolumeExpansion field
+
+	labels := map[string]string{
+		"app.kubernetes.io/component":  "pxc",
+		"app.kubernetes.io/instance":   cr.Name,
+		"app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator",
+		"app.kubernetes.io/name":       "percona-xtradb-cluster",
+		"app.kubernetes.io/part-of":    "percona-xtradb-cluster",
+	}
+
+	pvcList := corev1.PersistentVolumeClaimList{}
+	if err := r.client.List(ctx, &pvcList, client.InNamespace(cr.Namespace), client.MatchingLabels(labels)); err != nil {
+		return errors.Wrap(err, "list persistentvolumeclaims")
+	}
+
+	pvcNames := make([]string, 0, len(pvcList.Items))
+	for _, pvc := range pvcList.Items {
+		pvcNames = append(pvcNames, pvc.Name)
+	}
+
+	log.Info("Resizing PVCs", "requested", requested, "actual", actual, "pvcList", strings.Join(pvcNames, ","))
+
+	log.Info("Deleting statefulset", "name", sts.Name)
+
+	if err := r.client.Delete(ctx, sts, client.PropagationPolicy("Orphan")); err != nil {
+		return errors.Wrapf(err, "delete statefulset/%s", sts.Name)
+	}
+
+	for _, pvc := range pvcList.Items {
+		if !strings.HasPrefix(pvc.Name, "datadir-"+sts.Name) {
+			continue
+		}
+
+		log.Info("Resizing PVC", "name", pvc.Name)
+		pvc.Spec.Resources.Requests[corev1.ResourceStorage] = requested
+
+		if err := r.client.Update(ctx, &pvc); err != nil {
+			return errors.Wrapf(err, "update persistentvolumeclaim/%s", pvc.Name)
+		}
+	}
+
+	return nil
+}