Skip to content

Commit

Permalink
add ObservedGeneration and add partition handling, and update e2e test
Browse files Browse the repository at this point in the history 
Signed-off-by: YZ775 <yuzuki-mimura@cybozu.co.jp>
  • Loading branch information
@YZ775
YZ775 committed Jul 26, 2023
1 parent c2723bb commit 759adbb
Show file tree
Hide file tree
Showing 4 changed files with 126 additions and 26 deletions.
3 changes: 1 addition & 2 deletions clustering/process.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,15 +228,14 @@ func (p *managerProcess) updateStatus(ctx context.Context, ss *StatusSet) error
p.metrics.backupWarnings.Set(float64(len(bs.Warnings)))
}

now := metav1.Now()
ststr := ss.State.String()
updateCond := func(typ string, val metav1.ConditionStatus, current []metav1.Condition) metav1.Condition {
updated := metav1.Condition{
Type: typ,
Status: val,
ObservedGeneration: ss.Cluster.Generation,
Reason: ststr,
Message: "the current state is " + ststr,
LastTransitionTime: now,
}

for _, cond := range current {
Expand Down
22 changes: 17 additions & 5 deletions controllers/mysqlcluster_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1870,30 +1870,42 @@ func (r *MySQLClusterReconciler) UpdateStatus(ctx context.Context, req ctrl.Requ
needUpdate := false
updated := metav1.ConditionFalse
reason := "UpdateInProgress"
message := "the current state is UpdateInProgress"
if sts.Spec.UpdateStrategy.Type == appsv1.RollingUpdateStatefulSetStrategyType && sts.Spec.UpdateStrategy.RollingUpdate != nil {
if sts.Spec.Replicas != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
updated = metav1.ConditionFalse
reason = "UpdateInProgress"
message = "UpdateInProgress, Partition is enabled, waiting for the all pods to be updated"
if sts.Status.UpdatedReplicas < (*sts.Spec.Replicas - *sts.Spec.UpdateStrategy.RollingUpdate.Partition) {
message = "UpdateInProgress, Partition is enabled, waiting for partitioned roll out to finish"
}
}
}
if sts.Status.AvailableReplicas != 0 && sts.Status.CurrentRevision == sts.Status.UpdateRevision && sts.Generation == sts.Status.ObservedGeneration {
updated = metav1.ConditionTrue
reason = "UpdateCompleted"
message = "the current state is UpdateCompleted"
}
if cluster.Status.ReconcileInfo.Generation != cluster.Generation {
cluster.Status.ReconcileInfo.Generation = cluster.Generation
cluster.Status.ReconcileInfo.ReconcileVersion = 1
needUpdate = true
}
ConditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate)

currentConditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate)
// if current status and new status are different, update status
if ConditionUpToDate == nil || ConditionUpToDate.Status != updated {
if currentConditionUpToDate == nil || currentConditionUpToDate.Status != updated || currentConditionUpToDate.Message != message {
needUpdate = true
}
if needUpdate {
cond := metav1.Condition{
Type: mocov1beta2.ConditionUpToDate,
Status: updated,
ObservedGeneration: cluster.Generation,
Reason: reason,
Message: "the current state is " + reason,
LastTransitionTime: metav1.Now(),
Message: message,
}
meta.SetStatusCondition(&cluster.Status.Conditions, cond)

if err := r.Status().Update(ctx, cluster); err != nil {
log.Error(err, "failed to update reconciliation info")
return err
Expand Down
119 changes: 103 additions & 16 deletions controllers/mysqlcluster_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1646,11 +1646,11 @@ var _ = Describe("MySQLCluster reconciler", func() {
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil {
return err
}
ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if ConditionUpToDate == nil {
conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if conditionUpToDate == nil {
return fmt.Errorf("not yet updated")
}
if ConditionUpToDate.Status != metav1.ConditionTrue {
if conditionUpToDate.Status != metav1.ConditionTrue {
return fmt.Errorf("not yet updated")
}
return nil
Expand Down Expand Up @@ -1689,11 +1689,11 @@ var _ = Describe("MySQLCluster reconciler", func() {
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil {
return err
}
ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if ConditionUpToDate == nil {
conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if conditionUpToDate == nil {
return fmt.Errorf("not yet updated")
}
if ConditionUpToDate.Status != metav1.ConditionTrue {
if conditionUpToDate.Status != metav1.ConditionTrue {
return fmt.Errorf("not yet updated")
}
return nil
Expand All @@ -1717,15 +1717,15 @@ var _ = Describe("MySQLCluster reconciler", func() {
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil {
return err
}
ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if ConditionUpToDate == nil {
conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if conditionUpToDate == nil {
return fmt.Errorf("not yet updated")
}
if err != nil {
return err
}
if ConditionUpToDate.Status != metav1.ConditionFalse {
fmt.Printf("condition: %+v\n", ConditionUpToDate)
if conditionUpToDate.Status != metav1.ConditionFalse {
fmt.Printf("condition: %+v\n", conditionUpToDate)
return fmt.Errorf("not yet updated")
}
return nil
Expand All @@ -1749,11 +1749,11 @@ var _ = Describe("MySQLCluster reconciler", func() {
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil {
return err
}
ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if ConditionUpToDate == nil {
conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if conditionUpToDate == nil {
return fmt.Errorf("not yet updated")
}
if ConditionUpToDate.Status != metav1.ConditionTrue {
if conditionUpToDate.Status != metav1.ConditionTrue {
return fmt.Errorf("not yet updated")
}
return nil
Expand All @@ -1777,14 +1777,101 @@ var _ = Describe("MySQLCluster reconciler", func() {
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil {
return err
}
ConditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if ConditionUpToDate == nil {
conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if conditionUpToDate == nil {
return fmt.Errorf("not yet updated")
}
if ConditionUpToDate.Status != metav1.ConditionFalse {
if conditionUpToDate.Status != metav1.ConditionFalse {
return fmt.Errorf("not yet updated")
}
return nil
}).Should(Succeed())
})

It("should sets reson to be expected when partition is enabled", func() {
cluster := testNewMySQLCluster("test")
err := k8sClient.Create(ctx, cluster)
Expect(err).NotTo(HaveOccurred())

var sts *appsv1.StatefulSet
Eventually(func() error {
sts = &appsv1.StatefulSet{}
if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "moco-test"}, sts); err != nil {
return err
}
return nil
}).Should(Succeed())

By("setting sts status to be partitioned roll out in progress")
sts.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType
sts.Spec.UpdateStrategy.RollingUpdate = &appsv1.RollingUpdateStatefulSetStrategy{
Partition: pointer.Int32(2),
}
err = k8sClient.Update(ctx, sts)
Expect(err).NotTo(HaveOccurred())
sts.Status.Replicas = 3
sts.Status.ReadyReplicas = 3
sts.Status.AvailableReplicas = 3
sts.Status.CurrentRevision = "hoge"
sts.Status.UpdateRevision = "fuga"
sts.Status.CurrentReplicas = 3
sts.Status.UpdatedReplicas = 0
sts.Status.ObservedGeneration = sts.Generation
err = k8sClient.Status().Update(ctx, sts)
Expect(err).NotTo(HaveOccurred())

By("checking reason is expected")
Eventually(func() error {
cluster2 := &mocov1beta2.MySQLCluster{}
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil {
return err
}
conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if conditionUpToDate == nil {
return fmt.Errorf("not yet updated")
}
if conditionUpToDate.Message != "UpdateInProgress, Partition is enabled, waiting for partitioned roll out to finish" {
return fmt.Errorf("not yet updated: %s", conditionUpToDate.Message)
}
return nil
}).Should(Succeed())
cluster = &mocov1beta2.MySQLCluster{}
err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster)
Expect(err).NotTo(HaveOccurred())
conditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate)
Expect(conditionUpToDate.Status).To(Equal(metav1.ConditionFalse))

By("setting sts status to be partitioned roll out completed")
sts.Status.Replicas = 3
sts.Status.ReadyReplicas = 3
sts.Status.AvailableReplicas = 3
sts.Status.CurrentRevision = "hoge"
sts.Status.UpdateRevision = "fuga"
sts.Status.CurrentReplicas = 1
sts.Status.UpdatedReplicas = 2
sts.Status.ObservedGeneration = sts.Generation
err = k8sClient.Status().Update(ctx, sts)
Expect(err).NotTo(HaveOccurred())

By("checking reason is expected")
Eventually(func() error {
cluster2 := &mocov1beta2.MySQLCluster{}
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil {
return err
}
conditionUpToDate := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionUpToDate)
if conditionUpToDate == nil {
return fmt.Errorf("not yet updated")
}
if conditionUpToDate.Message != "UpdateInProgress, Partition is enabled, waiting for the all pods to be updated" {
return fmt.Errorf("not yet updated: %s", conditionUpToDate.Message)
}
return nil
}).Should(Succeed())
cluster = &mocov1beta2.MySQLCluster{}
err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster)
Expect(err).NotTo(HaveOccurred())
conditionUpToDate = meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate)
Expect(conditionUpToDate.Status).To(Equal(metav1.ConditionFalse))
})
})
8 changes: 5 additions & 3 deletions e2e/upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,17 @@ var _ = Context("upgrade", func() {
if err != nil {
return err
}
ConditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate)
if ConditionUpToDate == nil {
conditionUpToDate := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionUpToDate)
if conditionUpToDate == nil {
return fmt.Errorf("statefulset is not updated yet")
}
if ConditionUpToDate.Status != metav1.ConditionTrue {
if conditionUpToDate.Status != metav1.ConditionTrue {
return fmt.Errorf("statefulset is not updated yet")
}
return nil
}).Should(Succeed())
ConditionHealthy := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionHealthy)
Expect(ConditionHealthy).To(Equal(metav1.ConditionTrue))

})

Expand Down

0 comments on commit 759adbb

Please sign in to comment.