From 759adbb41e58bc29a194a924e5bcb319bdd568b0 Mon Sep 17 00:00:00 2001 From: YZ775 Date: Wed, 26 Jul 2023 08:27:54 +0000 Subject: [PATCH] add ObservedGeneration and add partition handling, and update e2e test Signed-off-by: YZ775 --- clustering/process.go | 3 +- controllers/mysqlcluster_controller.go | 22 +++- controllers/mysqlcluster_controller_test.go | 119 +++++++++++++++++--- e2e/upgrade_test.go | 8 +- 4 files changed, 126 insertions(+), 26 deletions(-) diff --git a/clustering/process.go b/clustering/process.go index 6dc84743b..f68ce1fef 100644 --- a/clustering/process.go +++ b/clustering/process.go @@ -228,15 +228,14 @@ func (p *managerProcess) updateStatus(ctx context.Context, ss *StatusSet) error p.metrics.backupWarnings.Set(float64(len(bs.Warnings))) } - now := metav1.Now() ststr := ss.State.String() updateCond := func(typ string, val metav1.ConditionStatus, current []metav1.Condition) metav1.Condition { updated := metav1.Condition{ Type: typ, Status: val, + ObservedGeneration: ss.Cluster.Generation, Reason: ststr, Message: "the current state is " + ststr, - LastTransitionTime: now, } for _, cond := range current { diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 539462b34..804424ebf 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -1870,30 +1870,42 @@ func (r *MySQLClusterReconciler) UpdateStatus(ctx context.Context, req ctrl.Requ needUpdate := false updated := metav1.ConditionFalse reason := "UpdateInProgress" + message := "the current state is UpdateInProgress" + if sts.Spec.UpdateStrategy.Type == appsv1.RollingUpdateStatefulSetStrategyType && sts.Spec.UpdateStrategy.RollingUpdate != nil { + if sts.Spec.Replicas != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil { + updated = metav1.ConditionFalse + reason = "UpdateInProgress" + message = "UpdateInProgress, Partition is enabled, waiting for the all pods to be updated" + if sts.Status.UpdatedReplicas < (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition) { + message = "UpdateInProgress, Partition is enabled, waiting for partitioned roll out to finish" + } + } + } if sts.Status.AvailableReplicas != 0 && sts.Status.CurrentRevision == sts.Status.UpdateRevision && sts.Generation == sts.Status.ObservedGeneration { updated = metav1.ConditionTrue reason = "UpdateCompleted" + message = "the current state is UpdateCompleted" } if cluster.Status.ReconcileInfo.Generation != cluster.Generation { cluster.Status.ReconcileInfo.Generation = cluster.Generation cluster.Status.ReconcileInfo.ReconcileVersion = 1 needUpdate = true } - ConditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate) + + currentConditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate) // if current status and new status are different, update status - if ConditionUpToDate == nil || ConditionUpToDate.Status != updated { + if currentConditionUpToDate == nil || currentConditionUpToDate.Status != updated || currentConditionUpToDate.Message != message { needUpdate = true } if needUpdate { cond := metav1.Condition{ Type: mocov1beta2.ConditionUpToDate, Status: updated, + ObservedGeneration: cluster.Generation, Reason: reason, - Message: "the current state is " + reason, - LastTransitionTime: metav1.Now(), + Message: message, } meta.SetStatusCondition(&cluster.Status.Conditions, cond) - if err := r.Status().Update(ctx, cluster); err != nil { log.Error(err, "failed to update reconciliation info") return err diff --git a/controllers/mysqlcluster_controller_test.go b/controllers/mysqlcluster_controller_test.go index 0b76cc7b7..e2e23bd8f 100644 --- a/controllers/mysqlcluster_controller_test.go +++ b/controllers/mysqlcluster_controller_test.go @@ -1646,11 +1646,11 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if ConditionUpToDate == nil { + conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) + if conditionUpToDate == nil { return fmt.Errorf("not yet updated") } - if ConditionUpToDate.Status != metav1.ConditionTrue { + if conditionUpToDate.Status != metav1.ConditionTrue { return fmt.Errorf("not yet updated") } return nil @@ -1689,11 +1689,11 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if ConditionUpToDate == nil { + conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) + if conditionUpToDate == nil { return fmt.Errorf("not yet updated") } - if ConditionUpToDate.Status != metav1.ConditionTrue { + if conditionUpToDate.Status != metav1.ConditionTrue { return fmt.Errorf("not yet updated") } return nil @@ -1717,15 +1717,15 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if ConditionUpToDate == nil { + conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) + if conditionUpToDate == nil { return fmt.Errorf("not yet updated") } if err != nil { return err } - if ConditionUpToDate.Status != metav1.ConditionFalse { - fmt.Printf("condition: %+v\n", ConditionUpToDate) + if conditionUpToDate.Status != metav1.ConditionFalse { + fmt.Printf("condition: %+v\n", conditionUpToDate) return fmt.Errorf("not yet updated") } return nil @@ -1749,11 +1749,11 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if ConditionUpToDate == nil { + conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) + if conditionUpToDate == nil { return fmt.Errorf("not yet updated") } - if ConditionUpToDate.Status != metav1.ConditionTrue { + if conditionUpToDate.Status != metav1.ConditionTrue { return fmt.Errorf("not yet updated") } return nil @@ -1777,14 +1777,101 @@ var _ = Describe("MySQLCluster reconciler", func() { if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { return err } - ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) - if ConditionUpToDate == nil { + conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) + if conditionUpToDate == nil { return fmt.Errorf("not yet updated") } - if ConditionUpToDate.Status != metav1.ConditionFalse { + if conditionUpToDate.Status != metav1.ConditionFalse { return fmt.Errorf("not yet updated") } return nil }).Should(Succeed()) }) + + It("should sets reson to be expected when partition is enabled", func() { + cluster := testNewMySQLCluster("test") + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + var sts *appsv1.StatefulSet + Eventually(func() error { + sts = &appsv1.StatefulSet{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "moco-test"}, sts); err != nil { + return err + } + return nil + }).Should(Succeed()) + + By("setting sts status to be partitioned roll out in progress") + sts.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType + sts.Spec.UpdateStrategy.RollingUpdate = &appsv1.RollingUpdateStatefulSetStrategy{ + Partition: pointer.Int32(2), + } + err = k8sClient.Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + sts.Status.AvailableReplicas = 3 + sts.Status.CurrentRevision = "hoge" + sts.Status.UpdateRevision = "fuga" + sts.Status.CurrentReplicas = 3 + sts.Status.UpdatedReplicas = 0 + sts.Status.ObservedGeneration = sts.Generation + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("checking reason is expected") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) + if conditionUpToDate == nil { + return fmt.Errorf("not yet updated") + } + if conditionUpToDate.Message != "UpdateInProgress, Partition is enabled, waiting for partitioned roll out to finish" { + return fmt.Errorf("not yet updated: %s", conditionUpToDate.Message) + } + return nil + }).Should(Succeed()) + cluster = &mocov1beta2.MySQLCluster{} + err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster) + Expect(err).NotTo(HaveOccurred()) + conditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate) + Expect(conditionUpToDate.Status).To(Equal(metav1.ConditionFalse)) + + By("setting sts status to be partitioned roll out completed") + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + sts.Status.AvailableReplicas = 3 + sts.Status.CurrentRevision = "hoge" + sts.Status.UpdateRevision = "fuga" + sts.Status.CurrentReplicas = 1 + sts.Status.UpdatedReplicas = 2 + sts.Status.ObservedGeneration = sts.Generation + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("checking reason is expected") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate) + if conditionUpToDate == nil { + return fmt.Errorf("not yet updated") + } + if conditionUpToDate.Message != "UpdateInProgress, Partition is enabled, waiting for the all pods to be updated" { + return fmt.Errorf("not yet updated: %s", conditionUpToDate.Message) + } + return nil + }).Should(Succeed()) + cluster = &mocov1beta2.MySQLCluster{} + err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster) + Expect(err).NotTo(HaveOccurred()) + conditionUpToDate = meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate) + Expect(conditionUpToDate.Status).To(Equal(metav1.ConditionFalse)) + }) }) diff --git a/e2e/upgrade_test.go b/e2e/upgrade_test.go index fc3bce7e3..e960183ae 100644 --- a/e2e/upgrade_test.go +++ b/e2e/upgrade_test.go @@ -155,15 +155,17 @@ var _ = Context("upgrade", func() { if err != nil { return err } - ConditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate) - if ConditionUpToDate == nil { + conditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate) + if conditionUpToDate == nil { return fmt.Errorf("statefulset is not updated yet") } - if ConditionUpToDate.Status != metav1.ConditionTrue { + if conditionUpToDate.Status != metav1.ConditionTrue { return fmt.Errorf("statefulset is not updated yet") } return nil }).Should(Succeed()) + ConditionHealthy := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionHealthy) + Expect(ConditionHealthy).To(Equal(metav1.ConditionTrue)) })