diff --git a/api/v1beta1/mysqlcluster_types.go b/api/v1beta1/mysqlcluster_types.go index e64897ed3..f78726ac2 100644 --- a/api/v1beta1/mysqlcluster_types.go +++ b/api/v1beta1/mysqlcluster_types.go @@ -288,10 +288,11 @@ type MySQLClusterStatus struct { } const ( - ConditionInitialized string = "Initialized" - ConditionAvailable string = "Available" - ConditionHealthy string = "Healthy" - ConditionUpToDate string = "UpToDate" + ConditionInitialized string = "Initialized" + ConditionAvailable string = "Available" + ConditionHealthy string = "Healthy" + ConditionStatefulSetReady string = "StatefulSetReady" + ConditionReconcileSuccess string = "ReconcileSuccess" ) // BackupStatus represents the status of the last successful backup. diff --git a/api/v1beta2/mysqlcluster_types.go b/api/v1beta2/mysqlcluster_types.go index 198ae5a13..3e89c3d31 100644 --- a/api/v1beta2/mysqlcluster_types.go +++ b/api/v1beta2/mysqlcluster_types.go @@ -602,10 +602,11 @@ type MySQLClusterStatus struct { } const ( - ConditionInitialized string = "Initialized" - ConditionAvailable string = "Available" - ConditionHealthy string = "Healthy" - ConditionUpToDate string = "UpToDate" + ConditionInitialized string = "Initialized" + ConditionAvailable string = "Available" + ConditionHealthy string = "Healthy" + ConditionStatefulSetReady string = "StatefulSetReady" + ConditionReconcileSuccess string = "ReconcileSuccess" ) // BackupStatus represents the status of the last successful backup. diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 666bba19f..f30c8f2f6 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -166,12 +166,23 @@ func (r *MySQLClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request reconciler = r.reconcileV1 } } - return reconciler(ctx, req, cluster) + result, err := reconciler(ctx, req, cluster) + if err2 := r.updateReconcileStatus(ctx, cluster, err); err2 != nil { + log.Error(err2, "failed to update reconcile status") + } + return result, err } -func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Request, cluster *mocov1beta2.MySQLCluster) (ctrl.Result, error) { +func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Request, cluster *mocov1beta2.MySQLCluster) (result ctrl.Result, err error) { log := crlog.FromContext(ctx) + defer func(ctx2 context.Context, cluster2 *mocov1beta2.MySQLCluster) { + if err2 := r.updateStatusByStatefulSet(ctx2, cluster2); err != nil { + err = err2 + log.Error(err2, "failed to update status") + } + }(ctx, cluster) + if cluster.DeletionTimestamp != nil { if !controllerutil.ContainsFinalizer(cluster, constants.MySQLClusterFinalizer) { return ctrl.Result{}, nil @@ -239,9 +250,6 @@ func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Reque log.Error(err, "failed to reconcile stateful set") return ctrl.Result{}, err } - if err := r.UpdateStatusByStatefulSet(ctx, cluster); err != nil { - return ctrl.Result{}, err - } if err := r.reconcileV1PDB(ctx, req, cluster); err != nil { return ctrl.Result{}, err @@ -1866,7 +1874,7 @@ func (r *MySQLClusterReconciler) finalizeV1(ctx context.Context, cluster *mocov1 return nil } -func (r *MySQLClusterReconciler) UpdateStatusByStatefulSet(ctx context.Context, cluster *mocov1beta2.MySQLCluster) error { +func (r *MySQLClusterReconciler) updateStatusByStatefulSet(ctx context.Context, cluster *mocov1beta2.MySQLCluster) error { log := crlog.FromContext(ctx) var sts appsv1.StatefulSet @@ -1875,33 +1883,51 @@ func (r *MySQLClusterReconciler) UpdateStatusByStatefulSet(ctx context.Context, return fmt.Errorf("failed to get StatefulSet %s/%s: %w", cluster.Namespace, cluster.PrefixedName(), err) } - updated := metav1.ConditionFalse - reason := "UpdateInProgress" - message := "the current state is UpdateInProgress" - if sts.Spec.UpdateStrategy.Type == appsv1.RollingUpdateStatefulSetStrategyType && sts.Spec.UpdateStrategy.RollingUpdate != nil { - if sts.Spec.Replicas != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil { - message = "UpdateInProgress, Partition is enabled, waiting for the all pods to be updated" - if sts.Status.UpdatedReplicas < (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition) { - message = "UpdateInProgress, Partition is enabled, waiting for partitioned roll out to finish" - } - } - } - if sts.Status.AvailableReplicas != 0 && sts.Status.CurrentRevision == sts.Status.UpdateRevision && sts.Generation == sts.Status.ObservedGeneration { - updated = metav1.ConditionTrue - reason = "UpdateCompleted" - message = "the current state is UpdateCompleted" + stsReady := metav1.ConditionFalse + reason := "StatefulSetNotReady" + if sts.Spec.Replicas == nil { + stsReady = metav1.ConditionFalse + reason = "StatefulSetNotReady" + } else if sts.Status.AvailableReplicas == *sts.Spec.Replicas && sts.Status.CurrentRevision == sts.Status.UpdateRevision && sts.Generation == sts.Status.ObservedGeneration { + stsReady = metav1.ConditionTrue + reason = "AllNewStatefulSetReady" } - currentConditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate) + currentConditionStatefulSetReady := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) // if current status and new status are different, update status - needUpdate := false - if currentConditionUpToDate == nil || currentConditionUpToDate.Status != updated || currentConditionUpToDate.Message != message { - needUpdate = true + if currentConditionStatefulSetReady == nil || currentConditionStatefulSetReady.Status != stsReady || currentConditionStatefulSetReady.ObservedGeneration != cluster.Generation { + cond := metav1.Condition{ + Type: mocov1beta2.ConditionStatefulSetReady, + Status: stsReady, + ObservedGeneration: cluster.Generation, + Reason: reason, + Message: "the current state is " + reason, + } + meta.SetStatusCondition(&cluster.Status.Conditions, cond) + if err := r.Status().Update(ctx, cluster); err != nil { + log.Error(err, "failed to update reconciliation info") + return err + } + log.Info("update status successfully") } - if needUpdate { + return nil +} + +func (r *MySQLClusterReconciler) updateReconcileStatus(ctx context.Context, cluster *mocov1beta2.MySQLCluster, reconcileErr error) error { + log := crlog.FromContext(ctx) + reason := "ReconcileFailed" + message := "reconcile failed" + success := metav1.ConditionFalse + if reconcileErr == nil { + reason = "ReconcileSuccess" + message = "reconcile successfully" + success = metav1.ConditionTrue + } + currentConditionReconcileSuccess := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionReconcileSuccess) + if currentConditionReconcileSuccess == nil || currentConditionReconcileSuccess.Status != success || currentConditionReconcileSuccess.ObservedGeneration != cluster.Generation { cond := metav1.Condition{ - Type: mocov1beta2.ConditionUpToDate, - Status: updated, + Type: mocov1beta2.ConditionReconcileSuccess, + Status: success, ObservedGeneration: cluster.Generation, Reason: reason, Message: message, diff --git a/controllers/mysqlcluster_controller_test.go b/controllers/mysqlcluster_controller_test.go index b0964ba12..1c0a3f1f5 100644 --- a/controllers/mysqlcluster_controller_test.go +++ b/controllers/mysqlcluster_controller_test.go @@ -1614,7 +1614,7 @@ var _ = Describe("MySQLCluster reconciler", func() { }).Should(Succeed()) }) - It("should sets ConditionUpToDate to be true when StatefulSet is ready", func() { + It("should sets ConditionStatefulSetReady to be true when StatefulSet is ready", func() { cluster := testNewMySQLCluster("test") err := k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) @@ -1646,18 +1646,18 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { return fmt.Errorf("not yet updated") } - if conditionUpToDate.Status != metav1.ConditionTrue { + if conditionStatefulSetReady.Status != metav1.ConditionTrue { return fmt.Errorf("not yet updated") } return nil }).Should(Succeed()) }) - It("should sets ConditionUpToDate to be false when status of StatefulSet is empty", func() { + It("should sets ConditionStatefulSetReady to be false when status of StatefulSet is empty", func() { cluster := testNewMySQLCluster("test") err := k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) @@ -1688,11 +1688,11 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { return fmt.Errorf("not yet updated") } - if conditionUpToDate.Status != metav1.ConditionTrue { + if conditionStatefulSetReady.Status != metav1.ConditionTrue { return fmt.Errorf("not yet updated") } return nil @@ -1709,18 +1709,18 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { return fmt.Errorf("condition does not exists") } - if conditionUpToDate.Status != metav1.ConditionFalse { + if conditionStatefulSetReady.Status != metav1.ConditionFalse { return fmt.Errorf("condition is not false") } return nil }).Should(Succeed()) }) - It("should not sets ConditionUpToDate to be true when StatefulSet is not ready", func() { + It("should not sets ConditionStatefulSetReady to be true when StatefulSet is not ready", func() { cluster := testNewMySQLCluster("test") err := k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) @@ -1752,11 +1752,11 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { return fmt.Errorf("not yet updated") } - if conditionUpToDate.Status != metav1.ConditionTrue { + if conditionStatefulSetReady.Status != metav1.ConditionTrue { return fmt.Errorf("not yet updated") } return nil @@ -1780,11 +1780,11 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { return fmt.Errorf("not yet updated") } - if conditionUpToDate.Status != metav1.ConditionFalse { + if conditionStatefulSetReady.Status != metav1.ConditionFalse { return fmt.Errorf("not yet updated") } return nil @@ -1808,11 +1808,11 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { return fmt.Errorf("not yet updated") } - if conditionUpToDate.Status != metav1.ConditionTrue { + if conditionStatefulSetReady.Status != metav1.ConditionTrue { return fmt.Errorf("not yet updated") } return nil @@ -1836,97 +1836,48 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { return fmt.Errorf("not yet updated") } - if conditionUpToDate.Status != metav1.ConditionFalse { + if conditionStatefulSetReady.Status != metav1.ConditionFalse { return fmt.Errorf("not yet updated") } return nil }).Should(Succeed()) }) - - It("should sets reson to be expected when partition is enabled", func() { - cluster := testNewMySQLCluster("test") - err := k8sClient.Create(ctx, cluster) - Expect(err).NotTo(HaveOccurred()) - - var sts *appsv1.StatefulSet - Eventually(func() error { - sts = &appsv1.StatefulSet{} - if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "moco-test"}, sts); err != nil { - return err - } - return nil - }).Should(Succeed()) - - By("setting sts status to be partitioned roll out in progress") - sts.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType - sts.Spec.UpdateStrategy.RollingUpdate = &appsv1.RollingUpdateStatefulSetStrategy{ - Partition: pointer.Int32(2), - } - err = k8sClient.Update(ctx, sts) - Expect(err).NotTo(HaveOccurred()) - sts.Status.Replicas = 3 - sts.Status.ReadyReplicas = 3 - sts.Status.AvailableReplicas = 3 - sts.Status.CurrentRevision = "hoge" - sts.Status.UpdateRevision = "fuga" - sts.Status.CurrentReplicas = 3 - sts.Status.UpdatedReplicas = 0 - sts.Status.ObservedGeneration = sts.Generation - err = k8sClient.Status().Update(ctx, sts) - Expect(err).NotTo(HaveOccurred()) - - By("checking reason is expected") - Eventually(func() error { - cluster2 := &mocov1beta2.MySQLCluster{} - if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { - return err - } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { - return fmt.Errorf("not yet updated") - } - if conditionUpToDate.Message != "UpdateInProgress, Partition is enabled, waiting for partitioned roll out to finish" { - return fmt.Errorf("not yet updated: %s", conditionUpToDate.Message) - } - if conditionUpToDate.Status != metav1.ConditionFalse { - return fmt.Errorf("status is not false") - } - return nil - }).Should(Succeed()) - - By("setting sts status to be partitioned roll out completed") - sts.Status.Replicas = 3 - sts.Status.ReadyReplicas = 3 - sts.Status.AvailableReplicas = 3 - sts.Status.CurrentRevision = "hoge" - sts.Status.UpdateRevision = "fuga" - sts.Status.CurrentReplicas = 1 - sts.Status.UpdatedReplicas = 2 - sts.Status.ObservedGeneration = sts.Generation - err = k8sClient.Status().Update(ctx, sts) - Expect(err).NotTo(HaveOccurred()) - - By("checking reason is expected") - Eventually(func() error { - cluster2 := &mocov1beta2.MySQLCluster{} - if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { - return err - } - conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { - return fmt.Errorf("not yet updated") - } - if conditionUpToDate.Message != "UpdateInProgress, Partition is enabled, waiting for the all pods to be updated" { - return fmt.Errorf("not yet updated: %s", conditionUpToDate.Message) - } - if conditionUpToDate.Status != metav1.ConditionFalse { - return fmt.Errorf("status is not false") - } - return nil - }).Should(Succeed()) - }) + // It("should sets reconcile status condition true when success", func() { + // cluster := testNewMySQLCluster("test") + // err := k8sClient.Create(ctx, cluster) + // Expect(err).NotTo(HaveOccurred()) + + // var sts *appsv1.StatefulSet + // Eventually(func() error { + // sts = &appsv1.StatefulSet{} + // if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "moco-test"}, sts); err != nil { + // return err + // } + // return nil + // }).Should(Succeed()) + // cluster = &mocov1beta2.MySQLCluster{} + // err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster) + // Expect(err).NotTo(HaveOccurred()) + // conditionReconcileSuccess := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionReconcileSuccess) + // Expect(conditionReconcileSuccess.Status).To(Equal(metav1.ConditionTrue)) + // }) + // It("should sets reconcile status condition false when success", func() { + // cluster := testNewMySQLCluster("test") + // err := k8sClient.Create(ctx, cluster) + // Expect(err).NotTo(HaveOccurred()) + + // cluster.Spec.MySQLConfigMapName = pointer.String("foo") + // err = k8sClient.Update(ctx, cluster) + // Expect(err).NotTo(HaveOccurred()) + + // cluster = &mocov1beta2.MySQLCluster{} + // err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster) + // Expect(err).NotTo(HaveOccurred()) + // conditionReconcileSuccess := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionReconcileSuccess) + // Expect(conditionReconcileSuccess.Status).To(Equal(metav1.ConditionFalse)) + // }) }) diff --git a/e2e/upgrade_test.go b/e2e/upgrade_test.go index fe394ba59..1941ee496 100644 --- a/e2e/upgrade_test.go +++ b/e2e/upgrade_test.go @@ -143,18 +143,19 @@ var _ = Context("upgrade", func() { return fmt.Errorf("cluster is not reconciled yet") } conditionHealthy := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionHealthy) + conditionStatefulSetReady := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("statefulset is not ready") + } if conditionHealthy == nil { return fmt.Errorf("cluster is not healthy") } if conditionHealthy.Status != metav1.ConditionTrue { + // Expect(conditionStatefulSetReady.Status).To(Equal(metav1.ConditionFalse)) return fmt.Errorf("cluster is not healthy") } - conditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate) - if conditionUpToDate == nil { - return fmt.Errorf("statefulset is not updated yet") - } - if conditionUpToDate.Status != metav1.ConditionTrue { - return fmt.Errorf("statefulset is not updated yet") + if conditionStatefulSetReady.Status != metav1.ConditionTrue { + return fmt.Errorf("statefulset is not ready") } return nil }).Should(Succeed())