From b7d698511aa80a7e8f3f6b8457476739230ffd14 Mon Sep 17 00:00:00 2001 From: YZ775 Date: Fri, 21 Jul 2023 09:09:55 +0000 Subject: [PATCH] add StatefulSet condition and change MySQLClusterCondition to metav1.Condition Signed-off-by: YZ775 add envtest Signed-off-by: YZ775 update gomod Signed-off-by: YZ775 update envtest and add e2e Signed-off-by: YZ775 update apidoc Signed-off-by: YZ775 rename Signed-off-by: YZ775 add ObservedGeneration and add partition handling, and update e2e test Signed-off-by: YZ775 remove unnecessary ObservedGeneration and move updateStatus() after reconcileV1StatefulSet() , and update e2e test Signed-off-by: YZ775 add document about UpToDate Signed-off-by: YZ775 update against review Signed-off-by: YZ775 wip change status to StatefulSetReady and add ReconcileSuccess, and fix unstable test Signed-off-by: YZ775 remove unnecessary condition check Signed-off-by: YZ775 remove redundant assignments in updateStatusByStatefulSet Signed-off-by: YZ775 move updateReconcileStatus in reconcileV1 Signed-off-by: YZ775 fix controller test Signed-off-by: YZ775 update documents Signed-off-by: YZ775 update lifecycle_test.go to check StatefulSetReady condition Signed-off-by: YZ775 fix auto generated document Signed-off-by: YZ775 join update status funcion Signed-off-by: YZ775 revert Eventually in envtest Signed-off-by: YZ775 fix multiple Update() call Signed-off-by: YZ775 update docs Signed-off-by: YZ775 fix controller Signed-off-by: YZ775 update condition when sts not found Signed-off-by: YZ775 update test Signed-off-by: YZ775 update lifecycle_test.go Signed-off-by: YZ775 fix condition Signed-off-by: YZ775 fix generation check and err handling Signed-off-by: YZ775 do not show err log when deletion of cluster Signed-off-by: YZ775 add StatefulSet condition and change MySQLClusterCondition to metav1.Condition Signed-off-by: YZ775 --- api/v1beta1/mysqlcluster_types.go | 35 +- api/v1beta1/zz_generated.conversion.go | 42 +-- api/v1beta1/zz_generated.deepcopy.go | 19 +- api/v1beta2/mysqlcluster_types.go | 35 +- api/v1beta2/zz_generated.deepcopy.go | 19 +- .../templates/generated/crds/moco_crds.yaml | 66 ++-- clustering/manager_test.go | 36 +- clustering/process.go | 55 ++-- .../bases/moco.cybozu.com_mysqlclusters.yaml | 66 ++-- ...inition_mysqlclusters.moco.cybozu.com.yaml | 66 ++-- controllers/mysqlcluster_controller.go | 107 ++++-- controllers/mysqlcluster_controller_test.go | 310 ++++++++++++++++++ docs/crd_mysqlcluster_v1beta1.md | 17 +- docs/crd_mysqlcluster_v1beta2.md | 17 +- docs/links.csv | 1 + docs/reconcile.md | 13 + e2e/backup_gcs_test.go | 5 +- e2e/backup_test.go | 5 +- e2e/failover_test.go | 7 +- e2e/failure_test.go | 5 +- e2e/lifecycle_test.go | 35 +- e2e/pvc_test.go | 5 +- e2e/replication_test.go | 13 +- e2e/upgrade_test.go | 39 ++- 24 files changed, 682 insertions(+), 336 deletions(-) diff --git a/api/v1beta1/mysqlcluster_types.go b/api/v1beta1/mysqlcluster_types.go index 313277bcb..f78726ac2 100644 --- a/api/v1beta1/mysqlcluster_types.go +++ b/api/v1beta1/mysqlcluster_types.go @@ -252,7 +252,7 @@ type MySQLClusterStatus struct { // Conditions is an array of conditions. // +optional - Conditions []MySQLClusterCondition `json:"conditions,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` // CurrentPrimaryIndex is the index of the current primary Pod in StatefulSet. // Initially, this is zero. @@ -287,35 +287,12 @@ type MySQLClusterStatus struct { ReconcileInfo ReconcileInfo `json:"reconcileInfo"` } -// MySQLClusterCondition defines the condition of MySQLCluster. -type MySQLClusterCondition struct { - // Type is the type of the condition. - Type MySQLClusterConditionType `json:"type"` - - // Status is the status of the condition. - Status corev1.ConditionStatus `json:"status"` - - // Reason is a one-word CamelCase reason for the condition's last transition. - // +optional - Reason string `json:"reason,omitempty"` - - // Message is a human-readable message indicating details about last transition. - // +optional - Message string `json:"message,omitempty"` - - // LastTransitionTime is the last time the condition transits from one status to another. - LastTransitionTime metav1.Time `json:"lastTransitionTime"` -} - -// MySQLClusterConditionType is the type of MySQLCluster condition. -// +kubebuilder:validation:Enum=Initialized;Available;Healthy -type MySQLClusterConditionType string - -// Valid values for MySQLClusterConditionType const ( - ConditionInitialized MySQLClusterConditionType = "Initialized" - ConditionAvailable MySQLClusterConditionType = "Available" - ConditionHealthy MySQLClusterConditionType = "Healthy" + ConditionInitialized string = "Initialized" + ConditionAvailable string = "Available" + ConditionHealthy string = "Healthy" + ConditionStatefulSetReady string = "StatefulSetReady" + ConditionReconcileSuccess string = "ReconcileSuccess" ) // BackupStatus represents the status of the last successful backup. diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index b160ab150..5168719be 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -116,16 +116,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*MySQLClusterCondition)(nil), (*v1beta2.MySQLClusterCondition)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert__MySQLClusterCondition_To_v1beta2_MySQLClusterCondition(a.(*MySQLClusterCondition), b.(*v1beta2.MySQLClusterCondition), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta2.MySQLClusterCondition)(nil), (*MySQLClusterCondition)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_MySQLClusterCondition_To__MySQLClusterCondition(a.(*v1beta2.MySQLClusterCondition), b.(*MySQLClusterCondition), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*MySQLClusterList)(nil), (*v1beta2.MySQLClusterList)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert__MySQLClusterList_To_v1beta2_MySQLClusterList(a.(*MySQLClusterList), b.(*v1beta2.MySQLClusterList), scope) }); err != nil { @@ -583,34 +573,6 @@ func autoConvert_v1beta2_MySQLCluster_To__MySQLCluster(in *v1beta2.MySQLCluster, return nil } -func autoConvert__MySQLClusterCondition_To_v1beta2_MySQLClusterCondition(in *MySQLClusterCondition, out *v1beta2.MySQLClusterCondition, s conversion.Scope) error { - out.Type = v1beta2.MySQLClusterConditionType(in.Type) - out.Status = corev1.ConditionStatus(in.Status) - out.Reason = in.Reason - out.Message = in.Message - out.LastTransitionTime = in.LastTransitionTime - return nil -} - -// Convert__MySQLClusterCondition_To_v1beta2_MySQLClusterCondition is an autogenerated conversion function. -func Convert__MySQLClusterCondition_To_v1beta2_MySQLClusterCondition(in *MySQLClusterCondition, out *v1beta2.MySQLClusterCondition, s conversion.Scope) error { - return autoConvert__MySQLClusterCondition_To_v1beta2_MySQLClusterCondition(in, out, s) -} - -func autoConvert_v1beta2_MySQLClusterCondition_To__MySQLClusterCondition(in *v1beta2.MySQLClusterCondition, out *MySQLClusterCondition, s conversion.Scope) error { - out.Type = MySQLClusterConditionType(in.Type) - out.Status = corev1.ConditionStatus(in.Status) - out.Reason = in.Reason - out.Message = in.Message - out.LastTransitionTime = in.LastTransitionTime - return nil -} - -// Convert_v1beta2_MySQLClusterCondition_To__MySQLClusterCondition is an autogenerated conversion function. -func Convert_v1beta2_MySQLClusterCondition_To__MySQLClusterCondition(in *v1beta2.MySQLClusterCondition, out *MySQLClusterCondition, s conversion.Scope) error { - return autoConvert_v1beta2_MySQLClusterCondition_To__MySQLClusterCondition(in, out, s) -} - func autoConvert__MySQLClusterList_To_v1beta2_MySQLClusterList(in *MySQLClusterList, out *v1beta2.MySQLClusterList, s conversion.Scope) error { out.ListMeta = in.ListMeta if in.Items != nil { @@ -695,7 +657,7 @@ func autoConvert_v1beta2_MySQLClusterSpec_To__MySQLClusterSpec(in *v1beta2.MySQL } func autoConvert__MySQLClusterStatus_To_v1beta2_MySQLClusterStatus(in *MySQLClusterStatus, out *v1beta2.MySQLClusterStatus, s conversion.Scope) error { - out.Conditions = *(*[]v1beta2.MySQLClusterCondition)(unsafe.Pointer(&in.Conditions)) + out.Conditions = *(*[]metav1.Condition)(unsafe.Pointer(&in.Conditions)) out.CurrentPrimaryIndex = in.CurrentPrimaryIndex out.SyncedReplicas = in.SyncedReplicas out.ErrantReplicas = in.ErrantReplicas @@ -717,7 +679,7 @@ func Convert__MySQLClusterStatus_To_v1beta2_MySQLClusterStatus(in *MySQLClusterS } func autoConvert_v1beta2_MySQLClusterStatus_To__MySQLClusterStatus(in *v1beta2.MySQLClusterStatus, out *MySQLClusterStatus, s conversion.Scope) error { - out.Conditions = *(*[]MySQLClusterCondition)(unsafe.Pointer(&in.Conditions)) + out.Conditions = *(*[]metav1.Condition)(unsafe.Pointer(&in.Conditions)) out.CurrentPrimaryIndex = in.CurrentPrimaryIndex out.SyncedReplicas = in.SyncedReplicas out.ErrantReplicas = in.ErrantReplicas diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0c0c308cf..4f4dad4d2 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1beta1 import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -249,22 +250,6 @@ func (in *MySQLCluster) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MySQLClusterCondition) DeepCopyInto(out *MySQLClusterCondition) { - *out = *in - in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MySQLClusterCondition. -func (in *MySQLClusterCondition) DeepCopy() *MySQLClusterCondition { - if in == nil { - return nil - } - out := new(MySQLClusterCondition) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MySQLClusterList) DeepCopyInto(out *MySQLClusterList) { *out = *in @@ -360,7 +345,7 @@ func (in *MySQLClusterStatus) DeepCopyInto(out *MySQLClusterStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]MySQLClusterCondition, len(*in)) + *out = make([]v1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/api/v1beta2/mysqlcluster_types.go b/api/v1beta2/mysqlcluster_types.go index 67ad3a8bd..3e89c3d31 100644 --- a/api/v1beta2/mysqlcluster_types.go +++ b/api/v1beta2/mysqlcluster_types.go @@ -566,7 +566,7 @@ type MySQLClusterStatus struct { // Conditions is an array of conditions. // +optional - Conditions []MySQLClusterCondition `json:"conditions,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` // CurrentPrimaryIndex is the index of the current primary Pod in StatefulSet. // Initially, this is zero. @@ -601,35 +601,12 @@ type MySQLClusterStatus struct { ReconcileInfo ReconcileInfo `json:"reconcileInfo"` } -// MySQLClusterCondition defines the condition of MySQLCluster. -type MySQLClusterCondition struct { - // Type is the type of the condition. - Type MySQLClusterConditionType `json:"type"` - - // Status is the status of the condition. - Status corev1.ConditionStatus `json:"status"` - - // Reason is a one-word CamelCase reason for the condition's last transition. - // +optional - Reason string `json:"reason,omitempty"` - - // Message is a human-readable message indicating details about last transition. - // +optional - Message string `json:"message,omitempty"` - - // LastTransitionTime is the last time the condition transits from one status to another. - LastTransitionTime metav1.Time `json:"lastTransitionTime"` -} - -// MySQLClusterConditionType is the type of MySQLCluster condition. -// +kubebuilder:validation:Enum=Initialized;Available;Healthy -type MySQLClusterConditionType string - -// Valid values for MySQLClusterConditionType const ( - ConditionInitialized MySQLClusterConditionType = "Initialized" - ConditionAvailable MySQLClusterConditionType = "Available" - ConditionHealthy MySQLClusterConditionType = "Healthy" + ConditionInitialized string = "Initialized" + ConditionAvailable string = "Available" + ConditionHealthy string = "Healthy" + ConditionStatefulSetReady string = "StatefulSetReady" + ConditionReconcileSuccess string = "ReconcileSuccess" ) // BackupStatus represents the status of the last successful backup. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index fdaf72972..b8a55d61f 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1beta2 import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -249,22 +250,6 @@ func (in *MySQLCluster) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MySQLClusterCondition) DeepCopyInto(out *MySQLClusterCondition) { - *out = *in - in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MySQLClusterCondition. -func (in *MySQLClusterCondition) DeepCopy() *MySQLClusterCondition { - if in == nil { - return nil - } - out := new(MySQLClusterCondition) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MySQLClusterList) DeepCopyInto(out *MySQLClusterList) { *out = *in @@ -365,7 +350,7 @@ func (in *MySQLClusterStatus) DeepCopyInto(out *MySQLClusterStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]MySQLClusterCondition, len(*in)) + *out = make([]v1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/charts/moco/templates/generated/crds/moco_crds.yaml b/charts/moco/templates/generated/crds/moco_crds.yaml index e355017a0..72eb87c2e 100644 --- a/charts/moco/templates/generated/crds/moco_crds.yaml +++ b/charts/moco/templates/generated/crds/moco_crds.yaml @@ -9596,30 +9596,43 @@ spec: conditions: description: Conditions is an array of conditions. items: - description: MySQLClusterCondition defines the condition of MyS + description: Condition contains details for one aspect of the c properties: lastTransitionTime: - description: 'LastTransitionTime is the last time the condition ' + description: 'lastTransitionTime is the last time the condition ' format: date-time type: string message: - description: Message is a human-readable message indicating det + description: message is a human readable message indicating det + maxLength: 32768 type: string + observedGeneration: + description: observedGeneration represents the .metadata. + format: int64 + minimum: 0 + type: integer reason: - description: Reason is a one-word CamelCase reason for the cond + description: reason contains a programmatic identifier indicati + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status is the status of the condition. + description: status of the condition, one of True, False, Unkno + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type is the type of the condition. - enum: - - Initialized - - Available - - Healthy + description: type of condition in CamelCase or in foo.example. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - lastTransitionTime + - message + - reason - status - type type: object @@ -15339,30 +15352,43 @@ spec: conditions: description: Conditions is an array of conditions. items: - description: MySQLClusterCondition defines the condition of MyS + description: Condition contains details for one aspect of the c properties: lastTransitionTime: - description: 'LastTransitionTime is the last time the condition ' + description: 'lastTransitionTime is the last time the condition ' format: date-time type: string message: - description: Message is a human-readable message indicating det + description: message is a human readable message indicating det + maxLength: 32768 type: string + observedGeneration: + description: observedGeneration represents the .metadata. + format: int64 + minimum: 0 + type: integer reason: - description: Reason is a one-word CamelCase reason for the cond + description: reason contains a programmatic identifier indicati + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status is the status of the condition. + description: status of the condition, one of True, False, Unkno + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type is the type of the condition. - enum: - - Initialized - - Available - - Healthy + description: type of condition in CamelCase or in foo.example. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - lastTransitionTime + - message + - reason - status - type type: object diff --git a/clustering/manager_test.go b/clustering/manager_test.go index e2287a3b0..819eff9ef 100644 --- a/clustering/manager_test.go +++ b/clustering/manager_test.go @@ -176,7 +176,7 @@ var _ = Describe("manager", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("not healthy") @@ -188,9 +188,9 @@ var _ = Describe("manager", func() { for _, cond := range cluster.Status.Conditions { switch cond.Type { case mocov1beta2.ConditionAvailable: - isAvailable = cond.Status == corev1.ConditionTrue + isAvailable = cond.Status == metav1.ConditionTrue case mocov1beta2.ConditionInitialized: - isInitialized = cond.Status == corev1.ConditionTrue + isInitialized = cond.Status == metav1.ConditionTrue } } Expect(isAvailable).To(BeTrue()) @@ -233,7 +233,7 @@ var _ = Describe("manager", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionFalse { + if cond.Status == metav1.ConditionFalse { return nil } return fmt.Errorf("cluster is still healthy") @@ -244,9 +244,9 @@ var _ = Describe("manager", func() { for _, cond := range cluster.Status.Conditions { switch cond.Type { case mocov1beta2.ConditionAvailable: - isAvailable = cond.Status == corev1.ConditionTrue + isAvailable = cond.Status == metav1.ConditionTrue case mocov1beta2.ConditionInitialized: - isInitialized = cond.Status == corev1.ConditionTrue + isInitialized = cond.Status == metav1.ConditionTrue } } Expect(isAvailable).To(BeFalse()) @@ -274,7 +274,7 @@ var _ = Describe("manager", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - Expect(cond.Status).To(Equal(corev1.ConditionFalse)) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) } }) @@ -347,9 +347,9 @@ var _ = Describe("manager", func() { for _, cond := range cluster.Status.Conditions { switch cond.Type { case mocov1beta2.ConditionAvailable: - isAvailable = cond.Status == corev1.ConditionTrue + isAvailable = cond.Status == metav1.ConditionTrue case mocov1beta2.ConditionHealthy: - isHealthy = cond.Status == corev1.ConditionTrue + isHealthy = cond.Status == metav1.ConditionTrue } } Expect(isAvailable).To(BeFalse()) @@ -388,7 +388,7 @@ var _ = Describe("manager", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("not healthy") @@ -452,7 +452,7 @@ var _ = Describe("manager", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("not healthy") @@ -549,7 +549,7 @@ var _ = Describe("manager", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("not healthy") @@ -661,7 +661,7 @@ var _ = Describe("manager", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("not healthy") @@ -689,9 +689,9 @@ var _ = Describe("manager", func() { for _, cond := range cluster.Status.Conditions { switch cond.Type { case mocov1beta2.ConditionHealthy: - isHealthy = cond.Status == corev1.ConditionTrue + isHealthy = cond.Status == metav1.ConditionTrue case mocov1beta2.ConditionAvailable: - isAvailable = cond.Status == corev1.ConditionTrue + isAvailable = cond.Status == metav1.ConditionTrue } } Expect(isHealthy).To(BeFalse()) @@ -743,9 +743,9 @@ var _ = Describe("manager", func() { for _, cond := range cluster.Status.Conditions { switch cond.Type { case mocov1beta2.ConditionHealthy: - isHealthy = cond.Status == corev1.ConditionTrue + isHealthy = cond.Status == metav1.ConditionTrue case mocov1beta2.ConditionAvailable: - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return true } } @@ -812,7 +812,7 @@ var _ = Describe("manager", func() { if cond.Type != mocov1beta2.ConditionAvailable { continue } - if cond.Status != corev1.ConditionFalse { + if cond.Status != metav1.ConditionFalse { continue } return cond.Reason == StateLost.String() diff --git a/clustering/process.go b/clustering/process.go index 0d68b837d..baa0533d3 100644 --- a/clustering/process.go +++ b/clustering/process.go @@ -11,8 +11,8 @@ import ( "github.com/cybozu-go/moco/pkg/metrics" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" @@ -228,25 +228,13 @@ func (p *managerProcess) updateStatus(ctx context.Context, ss *StatusSet) error p.metrics.backupWarnings.Set(float64(len(bs.Warnings))) } - now := metav1.Now() ststr := ss.State.String() - updateCond := func(typ mocov1beta2.MySQLClusterConditionType, val corev1.ConditionStatus, current []mocov1beta2.MySQLClusterCondition) mocov1beta2.MySQLClusterCondition { - updated := mocov1beta2.MySQLClusterCondition{ - Type: typ, - Status: val, - Reason: ststr, - Message: "the current state is " + ststr, - LastTransitionTime: now, - } - - for _, cond := range current { - if cond.Type != typ { - continue - } - if cond.Status == val { - updated.LastTransitionTime = cond.LastTransitionTime - } - break + updateCond := func(typ string, val metav1.ConditionStatus) metav1.Condition { + updated := metav1.Condition{ + Type: typ, + Status: val, + Reason: ststr, + Message: "the current state is " + ststr, } return updated } @@ -258,33 +246,32 @@ func (p *managerProcess) updateStatus(ctx context.Context, ss *StatusSet) error } orig := cluster.DeepCopy() - initialized := corev1.ConditionTrue - available := corev1.ConditionFalse - healthy := corev1.ConditionFalse + initialized := metav1.ConditionTrue + available := metav1.ConditionFalse + healthy := metav1.ConditionFalse switch ss.State { case StateCloning, StateRestoring: - initialized = corev1.ConditionFalse + initialized = metav1.ConditionFalse case StateHealthy: - available = corev1.ConditionTrue - healthy = corev1.ConditionTrue + available = metav1.ConditionTrue + healthy = metav1.ConditionTrue case StateDegraded: - available = corev1.ConditionTrue + available = metav1.ConditionTrue case StateFailed: case StateLost: case StateIncomplete: } - conditions := []mocov1beta2.MySQLClusterCondition{ - updateCond(mocov1beta2.ConditionInitialized, initialized, cluster.Status.Conditions), - updateCond(mocov1beta2.ConditionAvailable, available, cluster.Status.Conditions), - updateCond(mocov1beta2.ConditionHealthy, healthy, cluster.Status.Conditions), - } - cluster.Status.Conditions = conditions - if available == corev1.ConditionTrue { + + meta.SetStatusCondition(&cluster.Status.Conditions, updateCond(mocov1beta2.ConditionInitialized, initialized)) + meta.SetStatusCondition(&cluster.Status.Conditions, updateCond(mocov1beta2.ConditionAvailable, available)) + meta.SetStatusCondition(&cluster.Status.Conditions, updateCond(mocov1beta2.ConditionHealthy, healthy)) + + if available == metav1.ConditionTrue { p.metrics.available.Set(1) } else { p.metrics.available.Set(0) } - if healthy == corev1.ConditionTrue { + if healthy == metav1.ConditionTrue { p.metrics.healthy.Set(1) } else { p.metrics.healthy.Set(0) diff --git a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml index 8eff1de88..ab0859112 100644 --- a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml +++ b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml @@ -6063,31 +6063,44 @@ spec: conditions: description: Conditions is an array of conditions. items: - description: MySQLClusterCondition defines the condition of MyS + description: Condition contains details for one aspect of the c properties: lastTransitionTime: - description: 'LastTransitionTime is the last time the condition ' + description: 'lastTransitionTime is the last time the condition ' format: date-time type: string message: - description: Message is a human-readable message indicating + description: message is a human readable message indicating det + maxLength: 32768 type: string + observedGeneration: + description: observedGeneration represents the .metadata. + format: int64 + minimum: 0 + type: integer reason: - description: Reason is a one-word CamelCase reason for the cond + description: reason contains a programmatic identifier indicati + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status is the status of the condition. + description: status of the condition, one of True, False, Unkno + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type is the type of the condition. - enum: - - Initialized - - Available - - Healthy + description: type of condition in CamelCase or in foo.example. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - lastTransitionTime + - message + - reason - status - type type: object @@ -12346,31 +12359,44 @@ spec: conditions: description: Conditions is an array of conditions. items: - description: MySQLClusterCondition defines the condition of MyS + description: Condition contains details for one aspect of the c properties: lastTransitionTime: - description: 'LastTransitionTime is the last time the condition ' + description: 'lastTransitionTime is the last time the condition ' format: date-time type: string message: - description: Message is a human-readable message indicating + description: message is a human readable message indicating det + maxLength: 32768 type: string + observedGeneration: + description: observedGeneration represents the .metadata. + format: int64 + minimum: 0 + type: integer reason: - description: Reason is a one-word CamelCase reason for the cond + description: reason contains a programmatic identifier indicati + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status is the status of the condition. + description: status of the condition, one of True, False, Unkno + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type is the type of the condition. - enum: - - Initialized - - Available - - Healthy + description: type of condition in CamelCase or in foo.example. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - lastTransitionTime + - message + - reason - status - type type: object diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml index 2bfea8ed6..c6d62c033 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml @@ -6073,31 +6073,44 @@ spec: conditions: description: Conditions is an array of conditions. items: - description: MySQLClusterCondition defines the condition of MyS + description: Condition contains details for one aspect of the c properties: lastTransitionTime: - description: 'LastTransitionTime is the last time the condition ' + description: 'lastTransitionTime is the last time the condition ' format: date-time type: string message: - description: Message is a human-readable message indicating + description: message is a human readable message indicating det + maxLength: 32768 type: string + observedGeneration: + description: observedGeneration represents the .metadata. + format: int64 + minimum: 0 + type: integer reason: - description: Reason is a one-word CamelCase reason for the cond + description: reason contains a programmatic identifier indicati + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status is the status of the condition. + description: status of the condition, one of True, False, Unkno + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type is the type of the condition. - enum: - - Initialized - - Available - - Healthy + description: type of condition in CamelCase or in foo.example. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - lastTransitionTime + - message + - reason - status - type type: object @@ -12356,31 +12369,44 @@ spec: conditions: description: Conditions is an array of conditions. items: - description: MySQLClusterCondition defines the condition of MyS + description: Condition contains details for one aspect of the c properties: lastTransitionTime: - description: 'LastTransitionTime is the last time the condition ' + description: 'lastTransitionTime is the last time the condition ' format: date-time type: string message: - description: Message is a human-readable message indicating + description: message is a human readable message indicating det + maxLength: 32768 type: string + observedGeneration: + description: observedGeneration represents the .metadata. + format: int64 + minimum: 0 + type: integer reason: - description: Reason is a one-word CamelCase reason for the cond + description: reason contains a programmatic identifier indicati + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status is the status of the condition. + description: status of the condition, one of True, False, Unkno + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type is the type of the condition. - enum: - - Initialized - - Available - - Healthy + description: type of condition in CamelCase or in foo.example. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - lastTransitionTime + - message + - reason - status - type type: object diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 7970d4078..62eb1089b 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -25,6 +25,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -168,7 +169,7 @@ func (r *MySQLClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return reconciler(ctx, req, cluster) } -func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Request, cluster *mocov1beta2.MySQLCluster) (ctrl.Result, error) { +func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Request, cluster *mocov1beta2.MySQLCluster) (result ctrl.Result, err error) { log := crlog.FromContext(ctx) if cluster.DeletionTimestamp != nil { @@ -180,13 +181,13 @@ func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Reque r.ClusterManager.Stop(req.NamespacedName) - if err := r.finalizeV1(ctx, cluster); err != nil { + if err = r.finalizeV1(ctx, cluster); err != nil { log.Error(err, "failed to finalize") return ctrl.Result{}, err } controllerutil.RemoveFinalizer(cluster, constants.MySQLClusterFinalizer) - if err := r.Update(ctx, cluster); err != nil { + if err = r.Update(ctx, cluster); err != nil { log.Error(err, "failed to remove finalizer") return ctrl.Result{}, err } @@ -196,17 +197,24 @@ func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Reque return ctrl.Result{}, nil } - if err := r.reconcileV1Secret(ctx, req, cluster); err != nil { + defer func() { + if err2 := r.updateStatus(ctx, cluster, err); err2 != nil { + err = err2 + log.Error(err2, "failed to update status") + } + }() + + if err = r.reconcileV1Secret(ctx, req, cluster); err != nil { log.Error(err, "failed to reconcile secret") return ctrl.Result{}, err } - if err := r.reconcileV1Certificate(ctx, req, cluster); err != nil { + if err = r.reconcileV1Certificate(ctx, req, cluster); err != nil { log.Error(err, "failed to reconcile certificate") return ctrl.Result{}, err } - if err := r.reconcileV1GRPCSecret(ctx, req, cluster); err != nil { + if err = r.reconcileV1GRPCSecret(ctx, req, cluster); err != nil { log.Error(err, "failed to reconcile gRPC secret") return ctrl.Result{}, err } @@ -217,49 +225,40 @@ func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } - if err := r.reconcileV1FluentBitConfigMap(ctx, req, cluster); err != nil { + if err = r.reconcileV1FluentBitConfigMap(ctx, req, cluster); err != nil { log.Error(err, "failed to reconcile config maps for fluent-bit") return ctrl.Result{}, err } - if err := r.reconcileV1ServiceAccount(ctx, req, cluster); err != nil { + if err = r.reconcileV1ServiceAccount(ctx, req, cluster); err != nil { return ctrl.Result{}, err } - if err := r.reconcileV1Service(ctx, req, cluster); err != nil { + if err = r.reconcileV1Service(ctx, req, cluster); err != nil { return ctrl.Result{}, err } - if err := r.reconcilePVC(ctx, req, cluster); err != nil { + if err = r.reconcilePVC(ctx, req, cluster); err != nil { return ctrl.Result{}, err } - if err := r.reconcileV1StatefulSet(ctx, req, cluster, mycnf); err != nil { + if err = r.reconcileV1StatefulSet(ctx, req, cluster, mycnf); err != nil { log.Error(err, "failed to reconcile stateful set") return ctrl.Result{}, err } - if err := r.reconcileV1PDB(ctx, req, cluster); err != nil { + if err = r.reconcileV1PDB(ctx, req, cluster); err != nil { return ctrl.Result{}, err } - if err := r.reconcileV1BackupJob(ctx, req, cluster); err != nil { + if err = r.reconcileV1BackupJob(ctx, req, cluster); err != nil { return ctrl.Result{}, err } - if err := r.reconcileV1RestoreJob(ctx, req, cluster); err != nil { + if err = r.reconcileV1RestoreJob(ctx, req, cluster); err != nil { return ctrl.Result{}, err } - if cluster.Status.ReconcileInfo.Generation != cluster.Generation { - cluster.Status.ReconcileInfo.Generation = cluster.Generation - cluster.Status.ReconcileInfo.ReconcileVersion = 1 - if err := r.Status().Update(ctx, cluster); err != nil { - log.Error(err, "failed to update reconciliation info") - return ctrl.Result{}, err - } - } - r.ClusterManager.Update(client.ObjectKeyFromObject(cluster), string(controller.ReconcileIDFromContext(ctx))) return ctrl.Result{}, nil } @@ -1862,6 +1861,68 @@ func (r *MySQLClusterReconciler) finalizeV1(ctx context.Context, cluster *mocov1 return nil } +func (r *MySQLClusterReconciler) updateStatus(ctx context.Context, cluster *mocov1beta2.MySQLCluster, reconcileErr error) error { + log := crlog.FromContext(ctx) + orig := cluster.DeepCopy() + + cluster.Status.ReconcileInfo.Generation = cluster.Generation + cluster.Status.ReconcileInfo.ReconcileVersion = 1 + + stsReady := metav1.ConditionFalse + reason := "StatefulSetNotReady" + message := "StatefulSet is not ready" + var sts appsv1.StatefulSet + err := r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.PrefixedName()}, &sts) + if err != nil && apierrors.IsNotFound(err) { + reason = "StatefulSetNotFound" + message = "StatefulSet not found" + log.Error(err, "StatefulSet not found", "namespace", cluster.Namespace, "name", cluster.PrefixedName()) + } else if err != nil { + reason = "FaildToGetStatefulSet" + message = "failed to get StatefulSet" + log.Error(err, "failed to get StatefulSet", "namespace", cluster.Namespace, "name", cluster.PrefixedName()) + } else if sts.Spec.Replicas != nil && sts.Status.AvailableReplicas == *sts.Spec.Replicas && sts.Status.CurrentRevision == sts.Status.UpdateRevision && sts.Generation == sts.Status.ObservedGeneration { + stsReady = metav1.ConditionTrue + reason = "StatefulSetReady" + message = "StatefulSet is ready" + } + meta.SetStatusCondition(&cluster.Status.Conditions, + metav1.Condition{ + Type: mocov1beta2.ConditionStatefulSetReady, + Status: stsReady, + ObservedGeneration: cluster.Generation, + Reason: reason, + Message: message, + }, + ) + + reconcileSuccess := metav1.ConditionFalse + reason = "ReconcileFailed" + message = "reconcile failed" + if reconcileErr == nil { + reconcileSuccess = metav1.ConditionTrue + reason = "ReconcileSuccess" + message = "reconcile successfully" + } + meta.SetStatusCondition(&cluster.Status.Conditions, + metav1.Condition{ + Type: mocov1beta2.ConditionReconcileSuccess, + Status: reconcileSuccess, + ObservedGeneration: cluster.Generation, + Reason: reason, + Message: message, + }, + ) + + if !equality.Semantic.DeepEqual(orig, cluster) { + if err := r.Status().Update(ctx, cluster); err != nil { + return err + } + log.Info("update status successfully") + } + return nil +} + func setControllerReferenceWithConfigMap(cluster *mocov1beta2.MySQLCluster, cm *corev1ac.ConfigMapApplyConfiguration, scheme *runtime.Scheme) error { gvk, err := apiutil.GVKForObject(cluster, scheme) if err != nil { diff --git a/controllers/mysqlcluster_controller_test.go b/controllers/mysqlcluster_controller_test.go index f4bcf2ea5..4b5914292 100644 --- a/controllers/mysqlcluster_controller_test.go +++ b/controllers/mysqlcluster_controller_test.go @@ -15,6 +15,7 @@ import ( policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -1612,4 +1613,313 @@ var _ = Describe("MySQLCluster reconciler", func() { return nil }).Should(Succeed()) }) + + It("should sets ConditionStatefulSetReady to be true when StatefulSet is ready", func() { + cluster := testNewMySQLCluster("test") + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + var sts *appsv1.StatefulSet + Eventually(func() error { + sts = &appsv1.StatefulSet{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "moco-test"}, sts); err != nil { + return err + } + return nil + }).Should(Succeed()) + + By("setting sts status to be ready") + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + sts.Status.AvailableReplicas = 3 + sts.Status.CurrentRevision = "hoge" + sts.Status.UpdateRevision = "hoge" + sts.Status.CurrentReplicas = 3 + sts.Status.UpdatedReplicas = 3 + sts.Status.ObservedGeneration = sts.Generation + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("checking condition is true") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("condition does not exists") + } + if conditionStatefulSetReady.Status != metav1.ConditionTrue { + return fmt.Errorf("condition is not false") + } + return nil + }).Should(Succeed()) + }) + + It("should sets ConditionStatefulSetReady to be false when status of StatefulSet is empty", func() { + cluster := testNewMySQLCluster("test") + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + var sts *appsv1.StatefulSet + Eventually(func() error { + sts = &appsv1.StatefulSet{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "moco-test"}, sts); err != nil { + return err + } + return nil + }).Should(Succeed()) + + By("setting sts status to be ready") + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + sts.Status.AvailableReplicas = 3 + sts.Status.CurrentRevision = "hoge" + sts.Status.UpdateRevision = "hoge" + sts.Status.CurrentReplicas = 3 + sts.Status.UpdatedReplicas = 3 + sts.Status.ObservedGeneration = sts.Generation + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("waiting condition to be updated") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("condition does not exists") + } + if conditionStatefulSetReady.Status != metav1.ConditionTrue { + return fmt.Errorf("condition is not true") + } + return nil + }).Should(Succeed()) + + By("setting sts status to be empty") + sts.Status = appsv1.StatefulSetStatus{} + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("checking condition is false") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("condition does not exists") + } + if conditionStatefulSetReady.Status != metav1.ConditionFalse { + return fmt.Errorf("condition is not false") + } + return nil + }).Should(Succeed()) + }) + + It("should sets ConditionStatefulSetReady to be false when status of StatefulSet does not found", func() { + cluster := testNewMySQLCluster("test") + cluster.Spec.MySQLConfigMapName = pointer.String("foobarhoge") + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + By("checking condition is false") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("condition does not exists") + } + if conditionStatefulSetReady.Status != metav1.ConditionFalse { + return fmt.Errorf("condition is not false") + } + if conditionStatefulSetReady.Reason != "StatefulSetNotFound" { + return fmt.Errorf("reason is not expected") + } + return nil + }).Should(Succeed()) + }) + + It("should sets ConditionStatefulSetReady to be false when StatefulSet is not ready", func() { + cluster := testNewMySQLCluster("test") + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + var sts *appsv1.StatefulSet + Eventually(func() error { + sts = &appsv1.StatefulSet{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "moco-test"}, sts); err != nil { + return err + } + return nil + }).Should(Succeed()) + + By("setting sts status to be ready") + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + sts.Status.AvailableReplicas = 3 + sts.Status.CurrentRevision = "hoge" + sts.Status.UpdateRevision = "hoge" + sts.Status.CurrentReplicas = 3 + sts.Status.UpdatedReplicas = 3 + sts.Status.ObservedGeneration = sts.Generation + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("waiting condition to be updated") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("condition does not exists") + } + if conditionStatefulSetReady.Status != metav1.ConditionTrue { + return fmt.Errorf("condition is not true") + } + return nil + }).Should(Succeed()) + + By("setting sts status to be not ready") + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + sts.Status.AvailableReplicas = 3 + sts.Status.CurrentRevision = "hoge" + sts.Status.UpdateRevision = "fuga" + sts.Status.CurrentReplicas = 2 + sts.Status.UpdatedReplicas = 1 + sts.Status.ObservedGeneration = sts.Generation + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("checking condition is false") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("condition does not exists") + } + if conditionStatefulSetReady.Status != metav1.ConditionFalse { + return fmt.Errorf("condition is not false") + } + return nil + }).Should(Succeed()) + + By("setting sts status to be ready") + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + sts.Status.AvailableReplicas = 3 + sts.Status.CurrentRevision = "hoge" + sts.Status.UpdateRevision = "hoge" + sts.Status.CurrentReplicas = 3 + sts.Status.UpdatedReplicas = 3 + sts.Status.ObservedGeneration = sts.Generation + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("waiting condition to be updated") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("condition does not exists") + } + if conditionStatefulSetReady.Status != metav1.ConditionTrue { + return fmt.Errorf("condition is not true") + } + return nil + }).Should(Succeed()) + + By("setting sts status to be not reconciled yet") + sts.Status.Replicas = 3 + sts.Status.ReadyReplicas = 3 + sts.Status.AvailableReplicas = 3 + sts.Status.CurrentRevision = "hoge" + sts.Status.UpdateRevision = "hoge" + sts.Status.CurrentReplicas = 3 + sts.Status.UpdatedReplicas = 3 + sts.Status.ObservedGeneration = sts.Generation - 1 + err = k8sClient.Status().Update(ctx, sts) + Expect(err).NotTo(HaveOccurred()) + + By("checking condition is false") + Eventually(func() error { + cluster2 := &mocov1beta2.MySQLCluster{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster2); err != nil { + return err + } + conditionStatefulSetReady := meta.FindStatusCondition(cluster2.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + if conditionStatefulSetReady == nil { + return fmt.Errorf("condition does not exists") + } + if conditionStatefulSetReady.Status != metav1.ConditionFalse { + return fmt.Errorf("condition is not false") + } + return nil + }).Should(Succeed()) + }) + + It("should sets reconcile status condition true when success", func() { + cluster := testNewMySQLCluster("test") + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + By("checking condition is true") + Eventually(func() error { + cluster = &mocov1beta2.MySQLCluster{} + if err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster); err != nil { + return err + } + conditionReconcileSuccess := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionReconcileSuccess) + if conditionReconcileSuccess == nil { + return fmt.Errorf("condition does not exists") + + } + if conditionReconcileSuccess.Status != metav1.ConditionTrue { + return fmt.Errorf("condition is not true") + } + return nil + }).Should(Succeed()) + }) + + It("should sets reconcile status condition false when faild", func() { + cluster := testNewMySQLCluster("test") + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + By("setting configmap name to be invalid") + cluster.Spec.MySQLConfigMapName = pointer.String("foobarhoge") + err = k8sClient.Update(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + By("checking condition is false") + Eventually(func() error { + cluster = &mocov1beta2.MySQLCluster{} + if err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster); err != nil { + return err + } + conditionReconcileSuccess := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionReconcileSuccess) + if conditionReconcileSuccess == nil { + return fmt.Errorf("condition does not exists") + } + if conditionReconcileSuccess.Status != metav1.ConditionFalse { + return fmt.Errorf("condition is not false") + } + return nil + }).Should(Succeed()) + }) }) diff --git a/docs/crd_mysqlcluster_v1beta1.md b/docs/crd_mysqlcluster_v1beta1.md index ed928e355..1dc2e63f3 100644 --- a/docs/crd_mysqlcluster_v1beta1.md +++ b/docs/crd_mysqlcluster_v1beta1.md @@ -6,7 +6,6 @@ ### Sub Resources * [BackupStatus](#backupstatus) -* [MySQLClusterCondition](#mysqlclustercondition) * [MySQLClusterList](#mysqlclusterlist) * [MySQLClusterSpec](#mysqlclusterspec) * [MySQLClusterStatus](#mysqlclusterstatus) @@ -50,20 +49,6 @@ MySQLCluster is the Schema for the mysqlclusters API [Back to Custom Resources](#custom-resources) -#### MySQLClusterCondition - -MySQLClusterCondition defines the condition of MySQLCluster. - -| Field | Description | Scheme | Required | -| ----- | ----------- | ------ | -------- | -| type | Type is the type of the condition. | [MySQLClusterConditionType](https://pkg.go.dev/github.com/cybozu-go/moco/api/v1beta1#MySQLClusterConditionType) | true | -| status | Status is the status of the condition. | [corev1.ConditionStatus](https://pkg.go.dev/k8s.io/api/core/v1#ConditionStatus) | true | -| reason | Reason is a one-word CamelCase reason for the condition's last transition. | string | false | -| message | Message is a human-readable message indicating details about last transition. | string | false | -| lastTransitionTime | LastTransitionTime is the last time the condition transits from one status to another. | [metav1.Time](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Time) | true | - -[Back to Custom Resources](#custom-resources) - #### MySQLClusterList MySQLClusterList contains a list of MySQLCluster @@ -104,7 +89,7 @@ MySQLClusterStatus defines the observed state of MySQLCluster | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | -| conditions | Conditions is an array of conditions. | [][MySQLClusterCondition](#mysqlclustercondition) | false | +| conditions | Conditions is an array of conditions. | [][metav1.Condition](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition) | false | | currentPrimaryIndex | CurrentPrimaryIndex is the index of the current primary Pod in StatefulSet. Initially, this is zero. | int | true | | syncedReplicas | SyncedReplicas is the number of synced instances including the primary. | int | false | | errantReplicas | ErrantReplicas is the number of instances that have errant transactions. | int | false | diff --git a/docs/crd_mysqlcluster_v1beta2.md b/docs/crd_mysqlcluster_v1beta2.md index 1687760c6..e4cc184b0 100644 --- a/docs/crd_mysqlcluster_v1beta2.md +++ b/docs/crd_mysqlcluster_v1beta2.md @@ -6,7 +6,6 @@ ### Sub Resources * [BackupStatus](#backupstatus) -* [MySQLClusterCondition](#mysqlclustercondition) * [MySQLClusterList](#mysqlclusterlist) * [MySQLClusterSpec](#mysqlclusterspec) * [MySQLClusterStatus](#mysqlclusterstatus) @@ -51,20 +50,6 @@ MySQLCluster is the Schema for the mysqlclusters API [Back to Custom Resources](#custom-resources) -#### MySQLClusterCondition - -MySQLClusterCondition defines the condition of MySQLCluster. - -| Field | Description | Scheme | Required | -| ----- | ----------- | ------ | -------- | -| type | Type is the type of the condition. | [MySQLClusterConditionType](https://pkg.go.dev/github.com/cybozu-go/moco/api/v1beta1#MySQLClusterConditionType) | true | -| status | Status is the status of the condition. | [corev1.ConditionStatus](https://pkg.go.dev/k8s.io/api/core/v1#ConditionStatus) | true | -| reason | Reason is a one-word CamelCase reason for the condition's last transition. | string | false | -| message | Message is a human-readable message indicating details about last transition. | string | false | -| lastTransitionTime | LastTransitionTime is the last time the condition transits from one status to another. | [metav1.Time](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Time) | true | - -[Back to Custom Resources](#custom-resources) - #### MySQLClusterList MySQLClusterList contains a list of MySQLCluster @@ -106,7 +91,7 @@ MySQLClusterStatus defines the observed state of MySQLCluster | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | -| conditions | Conditions is an array of conditions. | [][MySQLClusterCondition](#mysqlclustercondition) | false | +| conditions | Conditions is an array of conditions. | [][metav1.Condition](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition) | false | | currentPrimaryIndex | CurrentPrimaryIndex is the index of the current primary Pod in StatefulSet. Initially, this is zero. | int | true | | syncedReplicas | SyncedReplicas is the number of synced instances including the primary. | int | false | | errantReplicas | ErrantReplicas is the number of instances that have errant transactions. | int | false | diff --git a/docs/links.csv b/docs/links.csv index d5402e203..74bc43602 100644 --- a/docs/links.csv +++ b/docs/links.csv @@ -7,6 +7,7 @@ corev1.PodSpec,https://pkg.go.dev/k8s.io/api/core/v1#PodSpec corev1.ServiceSpec,https://pkg.go.dev/k8s.io/api/core/v1#ServiceSpec corev1.VolumeSource,https://pkg.go.dev/k8s.io/api/core/v1#VolumeSource metav1.Duration,https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Duration +metav1.Condition,https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition metav1.ListMeta,https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ListMeta metav1.ObjectMeta,https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta metav1.Time,https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Time diff --git a/docs/reconcile.md b/docs/reconcile.md index 41c8c7b17..5e8770161 100644 --- a/docs/reconcile.md +++ b/docs/reconcile.md @@ -62,6 +62,13 @@ The StatefulSet will be updated when: The fluent-bit sidecar container is updated only when some fields under `spec` of MySQLCluster are modified. + +### Status about StatefulSet + +- In `MySQLCluster.Status.Condition`, there is a condition named `StatefulSetReady`. +- This indicates the readieness of StatefulSet. +- The condition will be `True` when the rolling update of StatefulSet completely finishes. + ### Secrets MOCO generates random passwords for users that MOCO uses to access MySQL. @@ -135,3 +142,9 @@ To restore data from a backup, MOCO creates a Job. MOCO deletes the Job after the Job finishes successfully. If the Job fails, MOCO leaves the Job. + +## Status of Reconcliation + +- In `MySQLCluster.Status.Condition`, there is a condition named `ReconcileSuccess`. +- This indicates the status of reconcilation. +- The condition will be `True` when the reconcile function successfully finishes. diff --git a/e2e/backup_gcs_test.go b/e2e/backup_gcs_test.go index 9b33f53f7..c5900aeff 100644 --- a/e2e/backup_gcs_test.go +++ b/e2e/backup_gcs_test.go @@ -16,6 +16,7 @@ import ( . "github.com/onsi/gomega" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //go:embed testdata/backup_gcs.yaml @@ -42,7 +43,7 @@ var _ = Context("backup-gcs", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -144,7 +145,7 @@ var _ = Context("backup-gcs", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } } diff --git a/e2e/backup_test.go b/e2e/backup_test.go index b22ec1b6a..d9e6cdffd 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -16,6 +16,7 @@ import ( . "github.com/onsi/gomega" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //go:embed testdata/backup.yaml @@ -48,7 +49,7 @@ var _ = Context("backup", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -150,7 +151,7 @@ var _ = Context("backup", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } } diff --git a/e2e/failover_test.go b/e2e/failover_test.go index c70157ae0..c91051f47 100644 --- a/e2e/failover_test.go +++ b/e2e/failover_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //go:embed testdata/failover.yaml @@ -31,7 +32,7 @@ var _ = Context("failure", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -79,7 +80,7 @@ var _ = Context("failure", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { currentPrimaryIndex = cluster.Status.CurrentPrimaryIndex return nil } @@ -126,7 +127,7 @@ var _ = Context("failure", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) diff --git a/e2e/failure_test.go b/e2e/failure_test.go index 94496837a..3f3e4d019 100644 --- a/e2e/failure_test.go +++ b/e2e/failure_test.go @@ -13,6 +13,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //go:embed testdata/failure.yaml @@ -34,7 +35,7 @@ var _ = Context("failure", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -83,7 +84,7 @@ var _ = Context("failure", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) diff --git a/e2e/lifecycle_test.go b/e2e/lifecycle_test.go index aa21cfc31..41986dcd7 100644 --- a/e2e/lifecycle_test.go +++ b/e2e/lifecycle_test.go @@ -15,6 +15,8 @@ import ( "github.com/prometheus/common/expfmt" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //go:embed testdata/single.yaml @@ -32,16 +34,31 @@ var _ = Context("lifecycle", func() { if err != nil { return err } - for _, cond := range cluster.Status.Conditions { - if cond.Type != mocov1beta2.ConditionHealthy { - continue - } - if cond.Status == corev1.ConditionTrue { - return nil - } - return fmt.Errorf("cluster is not healthy: %s", cond.Status) + if cluster.Generation != cluster.Status.ReconcileInfo.Generation { + return fmt.Errorf("cluster is not reconciled yet") + } + conditionReconcileSuccess := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionReconcileSuccess) + conditionStatefulSetReady := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + conditionHealthy := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionHealthy) + if conditionReconcileSuccess == nil { + return fmt.Errorf("reconcile failed") + } + if conditionStatefulSetReady == nil { + return fmt.Errorf("statefulset is not ready") + } + if conditionHealthy == nil { + return fmt.Errorf("cluster is not healthy") } - return errors.New("no health condition") + if conditionReconcileSuccess.Status != metav1.ConditionTrue { + return fmt.Errorf("condition ReconcileSuccess is not expected") + } + if conditionStatefulSetReady.Status != metav1.ConditionTrue { + return fmt.Errorf("condition StatefulSetReady is not expected") + } + if conditionHealthy.Status != metav1.ConditionTrue { + return fmt.Errorf("condition Healthy is not expected") + } + return nil }).Should(Succeed()) }) diff --git a/e2e/pvc_test.go b/e2e/pvc_test.go index 259568a8c..951865d15 100644 --- a/e2e/pvc_test.go +++ b/e2e/pvc_test.go @@ -15,6 +15,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //go:embed testdata/pvc_test.yaml @@ -36,7 +37,7 @@ var _ = Context("pvc_test", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -56,7 +57,7 @@ var _ = Context("pvc_test", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) diff --git a/e2e/replication_test.go b/e2e/replication_test.go index 523ac5f63..796a7d035 100644 --- a/e2e/replication_test.go +++ b/e2e/replication_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //go:embed testdata/donor.yaml @@ -36,7 +37,7 @@ var _ = Context("replication", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -84,7 +85,7 @@ var _ = Context("replication", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -132,7 +133,7 @@ var _ = Context("replication", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -165,7 +166,7 @@ var _ = Context("replication", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -244,7 +245,7 @@ var _ = Context("replication", func() { if cond.Type != mocov1beta2.ConditionAvailable { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not available: %s", cond.Status) @@ -298,7 +299,7 @@ var _ = Context("replication", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) diff --git a/e2e/upgrade_test.go b/e2e/upgrade_test.go index d86fab6a7..454859677 100644 --- a/e2e/upgrade_test.go +++ b/e2e/upgrade_test.go @@ -11,6 +11,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) //go:embed testdata/upgrade.yaml @@ -38,7 +40,7 @@ var _ = Context("upgrade", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -93,7 +95,7 @@ var _ = Context("upgrade", func() { if cond.Type != mocov1beta2.ConditionHealthy { continue } - if cond.Status == corev1.ConditionTrue { + if cond.Status == metav1.ConditionTrue { return nil } return fmt.Errorf("cluster is not healthy: %s", cond.Status) @@ -136,16 +138,31 @@ var _ = Context("upgrade", func() { if err != nil { return err } - for _, cond := range cluster.Status.Conditions { - if cond.Type != mocov1beta2.ConditionHealthy { - continue - } - if cond.Status == corev1.ConditionTrue { - return nil - } - return fmt.Errorf("cluster is not healthy: %s", cond.Status) + if cluster.Generation != cluster.Status.ReconcileInfo.Generation { + return fmt.Errorf("cluster is not reconciled yet") } - return errors.New("no health condition") + conditionReconcileSuccess := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionReconcileSuccess) + conditionStatefulSetReady := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionStatefulSetReady) + conditionHealthy := meta.FindStatusCondition(cluster.Status.Conditions, mocov1beta2.ConditionHealthy) + if conditionReconcileSuccess == nil { + return fmt.Errorf("reconcile failed") + } + if conditionStatefulSetReady == nil { + return fmt.Errorf("statefulset is not ready") + } + if conditionHealthy == nil { + return fmt.Errorf("cluster is not healthy") + } + if conditionReconcileSuccess.Status != metav1.ConditionTrue { + return fmt.Errorf("condition ReconcileSuccess is not expected") + } + if conditionStatefulSetReady.Status != metav1.ConditionTrue { + return fmt.Errorf("condition StatefulSetReady is not expected") + } + if conditionHealthy.Status != metav1.ConditionTrue { + return fmt.Errorf("condition Healthy is not expected") + } + return nil }).Should(Succeed()) })