Skip to content

Commit

Permalink
K8SPSMDB-1117: Improve PVC resizing (#1588)
Browse files Browse the repository at this point in the history 
* K8SPSMDB-1117: Improve PVC resizing

* address review comments

* fix k8s version

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>
  • Loading branch information
@egegunes @hors
egegunes and hors authored Jul 16, 2024
1 parent f33646e commit 5a1f431
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 69 deletions.
5 changes: 0 additions & 5 deletions pkg/apis/psmdb/v1/psmdb_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1220,8 +1220,3 @@ const (
AnnotationResyncPBM = "percona.com/resync-pbm"
AnnotationPVCResizeInProgress = "percona.com/pvc-resize-in-progress"
)

func (cr *PerconaServerMongoDB) PVCResizeInProgress() bool {
_, ok := cr.Annotations[AnnotationPVCResizeInProgress]
return ok
}
150 changes: 86 additions & 64 deletions pkg/controller/perconaservermongodb/volumes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,30 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex
return nil
}

podList := corev1.PodList{}
if err := r.client.List(ctx, &podList, client.InNamespace(cr.Namespace), client.MatchingLabels(ls)); err != nil {
return errors.Wrap(err, "list pods")
}

podNames := make([]string, 0, len(podList.Items))
for _, pod := range podList.Items {
podNames = append(podNames, pod.Name)
}

pvcsToUpdate := make([]string, 0, len(pvcList.Items))
for _, pvc := range pvcList.Items {
if !validatePVCName(pvc, sts) {
continue
}

podName := strings.SplitN(pvc.Name, "-", 3)[2]
if !slices.Contains(podNames, podName) {
continue
}

pvcsToUpdate = append(pvcsToUpdate, pvc.Name)
}

var actual resource.Quantity
for _, pvc := range pvcList.Items {
if !validatePVCName(pvc, sts) {
Expand All @@ -74,26 +98,60 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex
}
}

if cr.PVCResizeInProgress() {
resizeStartedAt, err := time.Parse(time.RFC3339, cr.GetAnnotations()[psmdbv1.AnnotationPVCResizeInProgress])
if actual.IsZero() {
return nil
}

sts = sts.DeepCopy()
if err := r.client.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
return errors.Wrapf(err, "get statefulset %s", client.ObjectKeyFromObject(sts))
}

var volumeTemplate corev1.PersistentVolumeClaim
for _, vct := range sts.Spec.VolumeClaimTemplates {
if vct.Name == psmdb.MongodDataVolClaimName {
volumeTemplate = vct
}
}

requested := pvcSpec.Resources.Requests[corev1.ResourceStorage]
configured := volumeTemplate.Spec.Resources.Requests[corev1.ResourceStorage]

if requested.Format == resource.DecimalSI {
requested, err = resource.ParseQuantity(requested.String() + "i")
if err != nil {
return errors.Wrap(err, "parse requested storage size")
}
}

if sts.Annotations[psmdbv1.AnnotationPVCResizeInProgress] != "" {
resizeStartedAt, err := time.Parse(time.RFC3339, sts.GetAnnotations()[psmdbv1.AnnotationPVCResizeInProgress])
if err != nil {
return errors.Wrap(err, "parse annotation")
}

resizeInProgress := false
updatedPVCs := 0
for _, pvc := range pvcList.Items {
if !validatePVCName(pvc, sts) {
continue
}

if pvc.Status.Capacity.Storage().Cmp(requested) == 0 {
updatedPVCs++
log.Info("PVC resize finished", "name", pvc.Name, "size", pvc.Status.Capacity.Storage())
continue
}

for _, condition := range pvc.Status.Conditions {
if condition.Status != corev1.ConditionTrue {
continue
}

switch condition.Type {
case corev1.PersistentVolumeClaimResizing, corev1.PersistentVolumeClaimFileSystemResizePending:
resizeInProgress = true
log.V(1).Info(condition.Message, "pvc", pvc.Name, "type", condition.Type, "lastTransitionTime", condition.LastTransitionTime)
log.Info("PVC resize in progress", "pvc", pvc.Name, "lastTransitionTime", condition.LastTransitionTime)
}
Expand All @@ -108,14 +166,24 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex
}

for _, event := range events.Items {
if event.EventTime.Time.Before(resizeStartedAt) {
eventTime := event.EventTime.Time
if event.EventTime.IsZero() {
eventTime = event.DeprecatedFirstTimestamp.Time
}

if eventTime.Before(resizeStartedAt) {
continue
}

if event.Reason == "VolumeResizeFailed" {
log.Error(nil, "PVC resize failed", "reason", event.Reason, "message", event.Note)
switch event.Reason {
case "Resizing", "ExternalExpanding", "FileSystemResizeRequired":
log.Info("PVC resize in progress", "pvc", pvc.Name, "reason", event.Reason, "message", event.Note)
case "FileSystemResizeSuccessful":
log.Info("PVC resize completed", "pvc", pvc.Name, "reason", event.Reason, "message", event.Note)
case "VolumeResizeFailed":
log.Error(nil, "PVC resize failed", "pvc", pvc.Name, "reason", event.Reason, "message", event.Note)

if err := r.handlePVCResizeFailure(ctx, cr, sts, actual); err != nil {
if err := r.handlePVCResizeFailure(ctx, cr, sts, configured); err != nil {
return err
}

Expand All @@ -124,9 +192,14 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex
}
}

if !resizeInProgress {
if err := k8s.DeannotateObject(ctx, r.client, cr, psmdbv1.AnnotationPVCResizeInProgress); err != nil {
return errors.Wrap(err, "deannotate psmdb")
if updatedPVCs == len(pvcsToUpdate) {
log.Info("Deleting statefulset")

if err := r.client.Delete(ctx, sts, client.PropagationPolicy("Orphan")); err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
return errors.Wrapf(err, "delete statefulset/%s", sts.Name)
}

log.Info("PVC resize completed")
Expand All @@ -135,24 +208,6 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex
}
}

sts = sts.DeepCopy()
if err := r.client.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
return errors.Wrapf(err, "get statefulset %s", client.ObjectKeyFromObject(sts))
}

var volumeTemplate corev1.PersistentVolumeClaim
for _, vct := range sts.Spec.VolumeClaimTemplates {
if vct.Name == psmdb.MongodDataVolClaimName {
volumeTemplate = vct
}
}

requested := pvcSpec.Resources.Requests[corev1.ResourceStorage]
configured := volumeTemplate.Spec.Resources.Requests[corev1.ResourceStorage]

if requested.Cmp(actual) < 0 {
return errors.Errorf("requested storage (%s) is less than actual storage (%s)", requested.String(), actual.String())
}
Expand All @@ -161,35 +216,11 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex
return nil
}

err = k8s.AnnotateObject(ctx, r.client, cr, map[string]string{psmdbv1.AnnotationPVCResizeInProgress: metav1.Now().Format(time.RFC3339)})
err = k8s.AnnotateObject(ctx, r.client, sts, map[string]string{psmdbv1.AnnotationPVCResizeInProgress: metav1.Now().Format(time.RFC3339)})
if err != nil {
return errors.Wrap(err, "annotate psmdb")
}

podList := corev1.PodList{}
if err := r.client.List(ctx, &podList, client.InNamespace(cr.Namespace), client.MatchingLabels(ls)); err != nil {
return errors.Wrap(err, "list pods")
}

podNames := make([]string, 0, len(podList.Items))
for _, pod := range podList.Items {
podNames = append(podNames, pod.Name)
}

pvcsToUpdate := make([]string, 0, len(pvcList.Items))
for _, pvc := range pvcList.Items {
if !validatePVCName(pvc, sts) {
continue
}

podName := strings.SplitN(pvc.Name, "-", 3)[2]
if !slices.Contains(podNames, podName) {
continue
}

pvcsToUpdate = append(pvcsToUpdate, pvc.Name)
}

log.Info("Resizing PVCs", "requested", requested, "actual", actual, "pvcList", strings.Join(pvcsToUpdate, ","))

for _, pvc := range pvcList.Items {
Expand Down Expand Up @@ -231,15 +262,6 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex
log.Info("PVC resize started", "pvc", pvc.Name, "requested", requested)
}

log.Info("Deleting statefulset")

if err := r.client.Delete(ctx, sts, client.PropagationPolicy("Orphan")); err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
return errors.Wrapf(err, "delete statefulset/%s", sts.Name)
}

return nil
}

Expand All @@ -248,7 +270,7 @@ func (r *ReconcilePerconaServerMongoDB) handlePVCResizeFailure(ctx context.Conte
return errors.Wrapf(err, "revert volume template for sts/%s", sts.Name)
}

if err := k8s.DeannotateObject(ctx, r.client, cr, psmdbv1.AnnotationPVCResizeInProgress); err != nil {
if err := k8s.DeannotateObject(ctx, r.client, sts, psmdbv1.AnnotationPVCResizeInProgress); err != nil {
return errors.Wrapf(err, "deannotate psmdb/%s", cr.Name)
}

Expand Down

0 comments on commit 5a1f431

Please sign in to comment.