diff --git a/pkg/apis/psmdb/v1/psmdb_types.go b/pkg/apis/psmdb/v1/psmdb_types.go index 3af1152fca..e8c2e85321 100644 --- a/pkg/apis/psmdb/v1/psmdb_types.go +++ b/pkg/apis/psmdb/v1/psmdb_types.go @@ -1220,8 +1220,3 @@ const ( AnnotationResyncPBM = "percona.com/resync-pbm" AnnotationPVCResizeInProgress = "percona.com/pvc-resize-in-progress" ) - -func (cr *PerconaServerMongoDB) PVCResizeInProgress() bool { - _, ok := cr.Annotations[AnnotationPVCResizeInProgress] - return ok -} \ No newline at end of file diff --git a/pkg/controller/perconaservermongodb/volumes.go b/pkg/controller/perconaservermongodb/volumes.go index aaa1397c54..a761b8c457 100644 --- a/pkg/controller/perconaservermongodb/volumes.go +++ b/pkg/controller/perconaservermongodb/volumes.go @@ -57,6 +57,30 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex return nil } + podList := corev1.PodList{} + if err := r.client.List(ctx, &podList, client.InNamespace(cr.Namespace), client.MatchingLabels(ls)); err != nil { + return errors.Wrap(err, "list pods") + } + + podNames := make([]string, 0, len(podList.Items)) + for _, pod := range podList.Items { + podNames = append(podNames, pod.Name) + } + + pvcsToUpdate := make([]string, 0, len(pvcList.Items)) + for _, pvc := range pvcList.Items { + if !validatePVCName(pvc, sts) { + continue + } + + podName := strings.SplitN(pvc.Name, "-", 3)[2] + if !slices.Contains(podNames, podName) { + continue + } + + pvcsToUpdate = append(pvcsToUpdate, pvc.Name) + } + var actual resource.Quantity for _, pvc := range pvcList.Items { if !validatePVCName(pvc, sts) { @@ -74,18 +98,53 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex } } - if cr.PVCResizeInProgress() { - resizeStartedAt, err := time.Parse(time.RFC3339, cr.GetAnnotations()[psmdbv1.AnnotationPVCResizeInProgress]) + if actual.IsZero() { + return nil + } + + sts = sts.DeepCopy() + if err := r.client.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return errors.Wrapf(err, "get statefulset %s", client.ObjectKeyFromObject(sts)) + } + + var volumeTemplate corev1.PersistentVolumeClaim + for _, vct := range sts.Spec.VolumeClaimTemplates { + if vct.Name == psmdb.MongodDataVolClaimName { + volumeTemplate = vct + } + } + + requested := pvcSpec.Resources.Requests[corev1.ResourceStorage] + configured := volumeTemplate.Spec.Resources.Requests[corev1.ResourceStorage] + + if requested.Format == resource.DecimalSI { + requested, err = resource.ParseQuantity(requested.String() + "i") + if err != nil { + return errors.Wrap(err, "parse requested storage size") + } + } + + if sts.Annotations[psmdbv1.AnnotationPVCResizeInProgress] != "" { + resizeStartedAt, err := time.Parse(time.RFC3339, sts.GetAnnotations()[psmdbv1.AnnotationPVCResizeInProgress]) if err != nil { return errors.Wrap(err, "parse annotation") } - resizeInProgress := false + updatedPVCs := 0 for _, pvc := range pvcList.Items { if !validatePVCName(pvc, sts) { continue } + if pvc.Status.Capacity.Storage().Cmp(requested) == 0 { + updatedPVCs++ + log.Info("PVC resize finished", "name", pvc.Name, "size", pvc.Status.Capacity.Storage()) + continue + } + for _, condition := range pvc.Status.Conditions { if condition.Status != corev1.ConditionTrue { continue @@ -93,7 +152,6 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex switch condition.Type { case corev1.PersistentVolumeClaimResizing, corev1.PersistentVolumeClaimFileSystemResizePending: - resizeInProgress = true log.V(1).Info(condition.Message, "pvc", pvc.Name, "type", condition.Type, "lastTransitionTime", condition.LastTransitionTime) log.Info("PVC resize in progress", "pvc", pvc.Name, "lastTransitionTime", condition.LastTransitionTime) } @@ -108,14 +166,24 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex } for _, event := range events.Items { - if event.EventTime.Time.Before(resizeStartedAt) { + eventTime := event.EventTime.Time + if event.EventTime.IsZero() { + eventTime = event.DeprecatedFirstTimestamp.Time + } + + if eventTime.Before(resizeStartedAt) { continue } - if event.Reason == "VolumeResizeFailed" { - log.Error(nil, "PVC resize failed", "reason", event.Reason, "message", event.Note) + switch event.Reason { + case "Resizing", "ExternalExpanding", "FileSystemResizeRequired": + log.Info("PVC resize in progress", "pvc", pvc.Name, "reason", event.Reason, "message", event.Note) + case "FileSystemResizeSuccessful": + log.Info("PVC resize completed", "pvc", pvc.Name, "reason", event.Reason, "message", event.Note) + case "VolumeResizeFailed": + log.Error(nil, "PVC resize failed", "pvc", pvc.Name, "reason", event.Reason, "message", event.Note) - if err := r.handlePVCResizeFailure(ctx, cr, sts, actual); err != nil { + if err := r.handlePVCResizeFailure(ctx, cr, sts, configured); err != nil { return err } @@ -124,9 +192,14 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex } } - if !resizeInProgress { - if err := k8s.DeannotateObject(ctx, r.client, cr, psmdbv1.AnnotationPVCResizeInProgress); err != nil { - return errors.Wrap(err, "deannotate psmdb") + if updatedPVCs == len(pvcsToUpdate) { + log.Info("Deleting statefulset") + + if err := r.client.Delete(ctx, sts, client.PropagationPolicy("Orphan")); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return errors.Wrapf(err, "delete statefulset/%s", sts.Name) } log.Info("PVC resize completed") @@ -135,24 +208,6 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex } } - sts = sts.DeepCopy() - if err := r.client.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - return errors.Wrapf(err, "get statefulset %s", client.ObjectKeyFromObject(sts)) - } - - var volumeTemplate corev1.PersistentVolumeClaim - for _, vct := range sts.Spec.VolumeClaimTemplates { - if vct.Name == psmdb.MongodDataVolClaimName { - volumeTemplate = vct - } - } - - requested := pvcSpec.Resources.Requests[corev1.ResourceStorage] - configured := volumeTemplate.Spec.Resources.Requests[corev1.ResourceStorage] - if requested.Cmp(actual) < 0 { return errors.Errorf("requested storage (%s) is less than actual storage (%s)", requested.String(), actual.String()) } @@ -161,35 +216,11 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex return nil } - err = k8s.AnnotateObject(ctx, r.client, cr, map[string]string{psmdbv1.AnnotationPVCResizeInProgress: metav1.Now().Format(time.RFC3339)}) + err = k8s.AnnotateObject(ctx, r.client, sts, map[string]string{psmdbv1.AnnotationPVCResizeInProgress: metav1.Now().Format(time.RFC3339)}) if err != nil { return errors.Wrap(err, "annotate psmdb") } - podList := corev1.PodList{} - if err := r.client.List(ctx, &podList, client.InNamespace(cr.Namespace), client.MatchingLabels(ls)); err != nil { - return errors.Wrap(err, "list pods") - } - - podNames := make([]string, 0, len(podList.Items)) - for _, pod := range podList.Items { - podNames = append(podNames, pod.Name) - } - - pvcsToUpdate := make([]string, 0, len(pvcList.Items)) - for _, pvc := range pvcList.Items { - if !validatePVCName(pvc, sts) { - continue - } - - podName := strings.SplitN(pvc.Name, "-", 3)[2] - if !slices.Contains(podNames, podName) { - continue - } - - pvcsToUpdate = append(pvcsToUpdate, pvc.Name) - } - log.Info("Resizing PVCs", "requested", requested, "actual", actual, "pvcList", strings.Join(pvcsToUpdate, ",")) for _, pvc := range pvcList.Items { @@ -231,15 +262,6 @@ func (r *ReconcilePerconaServerMongoDB) resizeVolumesIfNeeded(ctx context.Contex log.Info("PVC resize started", "pvc", pvc.Name, "requested", requested) } - log.Info("Deleting statefulset") - - if err := r.client.Delete(ctx, sts, client.PropagationPolicy("Orphan")); err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - return errors.Wrapf(err, "delete statefulset/%s", sts.Name) - } - return nil } @@ -248,7 +270,7 @@ func (r *ReconcilePerconaServerMongoDB) handlePVCResizeFailure(ctx context.Conte return errors.Wrapf(err, "revert volume template for sts/%s", sts.Name) } - if err := k8s.DeannotateObject(ctx, r.client, cr, psmdbv1.AnnotationPVCResizeInProgress); err != nil { + if err := k8s.DeannotateObject(ctx, r.client, sts, psmdbv1.AnnotationPVCResizeInProgress); err != nil { return errors.Wrapf(err, "deannotate psmdb/%s", cr.Name) }