From 57d81eda3178002e3797c988eb3e81e81a7dd252 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Wed, 16 Oct 2024 21:23:39 +0200 Subject: [PATCH] Add v1beta2 OwnerRemediated condition to KCP --- .../api/v1beta1/v1beta2_condition_consts.go | 62 +++++++- .../internal/controllers/remediation.go | 132 +++++++++++++++--- .../internal/controllers/remediation_test.go | 95 +++++++++++-- 3 files changed, 260 insertions(+), 29 deletions(-) diff --git a/controlplane/kubeadm/api/v1beta1/v1beta2_condition_consts.go b/controlplane/kubeadm/api/v1beta1/v1beta2_condition_consts.go index 81faa410f14a..40e82de59137 100644 --- a/controlplane/kubeadm/api/v1beta1/v1beta2_condition_consts.go +++ b/controlplane/kubeadm/api/v1beta1/v1beta2_condition_consts.go @@ -18,40 +18,100 @@ package v1beta1 import clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" -// Conditions that will be used for the KubeadmControlPlane object in v1Beta2 API version. +// KubeadmControlPlane's Available condition and corresponding reasons that will be used in v1Beta2 API version. const ( // KubeadmControlPlaneAvailableV1Beta2Condition True if the control plane can be reached, EtcdClusterAvailable is true, // and CertificatesAvailable is true. KubeadmControlPlaneAvailableV1Beta2Condition = clusterv1.AvailableV1Beta2Condition +) +// KubeadmControlPlane's CertificatesAvailable condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlaneCertificatesAvailableV1Beta2Condition True if all the cluster certificates exist. KubeadmControlPlaneCertificatesAvailableV1Beta2Condition = "CertificatesAvailable" + // KubeadmControlPlaneCertificatesInternalErrorV1Beta2Reason surfaces unexpected failures when reconciling cluster certificates. + KubeadmControlPlaneCertificatesInternalErrorV1Beta2Reason = clusterv1.InternalErrorV1Beta2Reason + + // KubeadmControlPlaneCertificatesAvailableV1Beta2Reason surfaces when cluster certificates are available, + // no matter if those certificates have been provided by the user or generated by KubeadmControlPlane itself. + // Cluster certificates includes: certificates authorities for ca, sa, front-proxy, etcd, and if external etcd is used, + // also the apiserver-etcd-client client certificate. + KubeadmControlPlaneCertificatesAvailableV1Beta2Reason = clusterv1.AvailableV1Beta2Condition +) + +// KubeadmControlPlane's EtcdClusterAvailable condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlaneEtcdClusterAvailableV1Beta2Condition surfaces issues to the managed etcd cluster, if any. // It is computed as aggregation of Machines's EtcdMemberHealthy (if not using an external etcd) conditions plus // additional checks validating potential issues to etcd quorum. KubeadmControlPlaneEtcdClusterAvailableV1Beta2Condition = "EtcdClusterAvailable" +) +// KubeadmControlPlane's MachinesReady condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlaneMachinesReadyV1Beta2Condition surfaces detail of issues on the controlled machines, if any. // Please note this will include also APIServerPodHealthy, ControllerManagerPodHealthy, SchedulerPodHealthy conditions. // If not using an external etcd also EtcdPodHealthy, EtcdMemberHealthy conditions are included. KubeadmControlPlaneMachinesReadyV1Beta2Condition = clusterv1.MachinesReadyV1Beta2Condition +) +// KubeadmControlPlane's MachinesUpToDate condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlaneMachinesUpToDateV1Beta2Condition surfaces details of controlled machines not up to date, if any. KubeadmControlPlaneMachinesUpToDateV1Beta2Condition = clusterv1.MachinesUpToDateV1Beta2Condition +) +// KubeadmControlPlane's ScalingUp condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlaneScalingUpV1Beta2Condition is true if available replicas < desired replicas. KubeadmControlPlaneScalingUpV1Beta2Condition = clusterv1.ScalingUpV1Beta2Condition +) +// KubeadmControlPlane's ScalingDown condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlaneScalingDownV1Beta2Condition is true if replicas > desired replicas. KubeadmControlPlaneScalingDownV1Beta2Condition = clusterv1.ScalingDownV1Beta2Condition +) +// KubeadmControlPlane's Remediating condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlaneRemediatingV1Beta2Condition surfaces details about ongoing remediation of the controlled machines, if any. KubeadmControlPlaneRemediatingV1Beta2Condition = clusterv1.RemediatingV1Beta2Condition +) + +// Reasons that will be used for the OwnerRemediated condition set by MachineHealthCheck on KubeadmControlPlane controlled machines +// being remediated in v1Beta2 API version. +const ( + // KubeadmControlPlaneMachineRemediationInternalErrorV1Beta2Reason surfaces unexpected failures while remediation a control plane machine. + KubeadmControlPlaneMachineRemediationInternalErrorV1Beta2Reason = clusterv1.InternalErrorV1Beta2Reason + + // KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason surfaces when remediation of a control plane machine can't be completed. + KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason = "CannotRemediate" + + // KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason surfaces when remediation of a control plane machine must be deferred. + KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason = "RemediationDeferred" + // KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason surfaces when remediation of a control plane machine has been completed. + // Note: After an unhealthy machine is deleted, a new one is created by the KubeadmControlPlaneMachine as part of the + // regular reconcile loop that ensures the correct number of replicas exist; KubeadmControlPlaneMachine waits for + // the new machine to exists before removing the controlplane.cluster.x-k8s.io/remediation-in-progress annotation. + // This is part of a series of safeguards to ensure that operation are performed sequentially on control plane machines. + KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason = "MachineDeleted" + + // KubeadmControlPlaneMachineRemediationDInternalErrorV1Beta2Reason surfaces unexpected failures when remediating + // a KubeadmControlPlane controlled machine. + KubeadmControlPlaneMachineRemediationDInternalErrorV1Beta2Reason = clusterv1.InternalErrorV1Beta2Reason +) + +// KubeadmControlPlane's Deleting condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlaneDeletingV1Beta2Condition surfaces details about ongoing deletion of the controlled machines. KubeadmControlPlaneDeletingV1Beta2Condition = clusterv1.DeletingV1Beta2Condition +) +// KubeadmControlPlane's Paused condition and corresponding reasons that will be used in v1Beta2 API version. +const ( // KubeadmControlPlanePausedV1Beta2Condition is true if this resource or the Cluster it belongs to are paused. KubeadmControlPlanePausedV1Beta2Condition = clusterv1.PausedV1Beta2Condition ) diff --git a/controlplane/kubeadm/internal/controllers/remediation.go b/controlplane/kubeadm/internal/controllers/remediation.go index e3a15668c593..5357c1670a4e 100644 --- a/controlplane/kubeadm/internal/controllers/remediation.go +++ b/controlplane/kubeadm/internal/controllers/remediation.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/collections" "sigs.k8s.io/cluster-api/util/conditions" + v1beta2conditions "sigs.k8s.io/cluster-api/util/conditions/v1beta2" "sigs.k8s.io/cluster-api/util/patch" ) @@ -49,22 +50,35 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C errList := []error{} healthyMachines := controlPlane.HealthyMachinesByMachineHealthCheck() for _, m := range healthyMachines { - if conditions.IsTrue(m, clusterv1.MachineHealthCheckSucceededCondition) && - conditions.IsFalse(m, clusterv1.MachineOwnerRemediatedCondition) && - m.DeletionTimestamp.IsZero() { - patchHelper, err := patch.NewHelper(m, r.Client) - if err != nil { - errList = append(errList, err) - continue - } + if !m.DeletionTimestamp.IsZero() { + continue + } + + shouldCleanup := conditions.IsTrue(m, clusterv1.MachineHealthCheckSucceededCondition) && conditions.IsFalse(m, clusterv1.MachineOwnerRemediatedCondition) + shouldCleanupV1Beta2 := v1beta2conditions.IsTrue(m, clusterv1.MachineHealthCheckSucceededV1Beta2Condition) && v1beta2conditions.IsFalse(m, clusterv1.MachineOwnerRemediatedV1Beta2Condition) + + if !(shouldCleanup || shouldCleanupV1Beta2) { + continue + } + + patchHelper, err := patch.NewHelper(m, r.Client) + if err != nil { + errList = append(errList, err) + continue + } + if shouldCleanup { conditions.Delete(m, clusterv1.MachineOwnerRemediatedCondition) + } - if err := patchHelper.Patch(ctx, m, patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ - clusterv1.MachineOwnerRemediatedCondition, - }}); err != nil { - errList = append(errList, err) - } + if shouldCleanupV1Beta2 { + v1beta2conditions.Delete(m, clusterv1.MachineOwnerRemediatedV1Beta2Condition) + } + + if err := patchHelper.Patch(ctx, m, patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ + clusterv1.MachineOwnerRemediatedCondition, + }}); err != nil { + errList = append(errList, err) } } if len(errList) > 0 { @@ -104,6 +118,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // of KCP reconcile. remediationData, err := RemediationDataFromAnnotation(v) if err != nil { + // TODO: Surface this in KCP's remediating condition return ctrl.Result{}, err } @@ -118,6 +133,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C } if !staleAnnotation { + // TODO: consider if to surface in KCP's remediating condition (this state should exists for a very short time) log.Info("Another remediation is already in progress. Skipping remediation.") return ctrl.Result{}, nil } @@ -125,14 +141,21 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C patchHelper, err := patch.NewHelper(machineToBeRemediated, r.Client) if err != nil { + log.Error(err, "failed to create patch helper") + return ctrl.Result{}, err } defer func() { // Always attempt to Patch the Machine conditions after each reconcileUnhealthyMachines. - if err := patchHelper.Patch(ctx, machineToBeRemediated, patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ - clusterv1.MachineOwnerRemediatedCondition, - }}); err != nil { + if err := patchHelper.Patch(ctx, machineToBeRemediated, + patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ + clusterv1.MachineOwnerRemediatedCondition, + }}, + patch.WithOwnedV1Beta2Conditions{Conditions: []string{ + clusterv1.MachineOwnerRemediatedV1Beta2Condition, + }}, + ); err != nil { log.Error(err, "Failed to patch control plane Machine", "Machine", machineToBeRemediated.Name) if retErr == nil { retErr = errors.Wrapf(err, "failed to patch control plane Machine %s", machineToBeRemediated.Name) @@ -164,6 +187,13 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C if controlPlane.Machines.Len() <= 1 { log.Info("A control plane machine needs remediation, but the number of current replicas is less or equal to 1. Skipping remediation", "replicas", controlPlane.Machines.Len()) conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate if current replicas are less or equal to 1") + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, + Message: "KCP can't remediate if current replicas are less or equal to 1", + }) return ctrl.Result{}, nil } @@ -171,6 +201,13 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C if controlPlane.HasHealthyMachineStillProvisioning() { log.Info("A control plane machine needs remediation, but there are other control-plane machines being provisioned. Skipping remediation") conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine provisioning to complete before triggering remediation") + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason, + Message: "KCP waiting for control plane machine provisioning to complete before triggering remediation", + }) return ctrl.Result{}, nil } @@ -178,6 +215,13 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C if controlPlane.HasDeletingMachine() { log.Info("A control plane machine needs remediation, but there are other control-plane machines being deleted. Skipping remediation") conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine deletion to complete before triggering remediation") + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason, + Message: "KCP waiting for control plane machine deletion to complete before triggering remediation", + }) return ctrl.Result{}, nil } @@ -187,11 +231,25 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C canSafelyRemediate, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, machineToBeRemediated) if err != nil { conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineRemediationInternalErrorV1Beta2Reason, + Message: "Please check controller logs for errors", + }) return ctrl.Result{}, err } if !canSafelyRemediate { log.Info("A control plane machine needs remediation, but removing this machine could result in etcd quorum loss. Skipping remediation") conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, + Message: "KCP can't remediate this machine because this could result in etcd loosing quorum", + }) return ctrl.Result{}, nil } } @@ -216,11 +274,25 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C log.Info("A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to") conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityWarning, "A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to. Skipping remediation") + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, + Message: "KCP can't remediate this machine because there is no healthy machine to forward etcd leadership to", + }) return ctrl.Result{}, nil } if err := workloadCluster.ForwardEtcdLeadership(ctx, machineToBeRemediated, etcdLeaderCandidate); err != nil { log.Error(err, "Failed to move etcd leadership to candidate machine", "candidate", klog.KObj(etcdLeaderCandidate)) conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineRemediationInternalErrorV1Beta2Reason, + Message: "Please check controller logs for errors", + }) return ctrl.Result{}, err } @@ -231,6 +303,13 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // Delete the machine if err := r.Client.Delete(ctx, machineToBeRemediated); err != nil { conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineRemediationInternalErrorV1Beta2Reason, + Message: "Please check controller logs for errors", + }) return ctrl.Result{}, errors.Wrapf(err, "failed to delete unhealthy machine %s", machineToBeRemediated.Name) } @@ -238,6 +317,12 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C log.Info("Remediating unhealthy machine") conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, + }) + // Prepare the info for tracking the remediation progress into the RemediationInProgressAnnotation. remediationInProgressValue, err := remediationInProgressData.Marshal() if err != nil { @@ -333,6 +418,13 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin if lastRemediationTime.Add(retryPeriod).After(reconciliationTime) { log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed in the latest %s. Skipping remediation", retryPeriod)) conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest %s (RetryPeriod)", retryPeriod) + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason, + Message: fmt.Sprintf("KCP can't remediate this machine because the operation already failed in the latest %s (RetryPeriod)", retryPeriod), + }) return remediationInProgressData, false, nil } @@ -342,6 +434,13 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin if remediationInProgressData.RetryCount >= maxRetry { log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed %d times (MaxRetry %d). Skipping remediation", remediationInProgressData.RetryCount, maxRetry)) conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed %d times (MaxRetry)", maxRetry) + + v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, + Message: fmt.Sprintf("KCP can't remediate this machine because the operation already failed %d times (MaxRetry)", maxRetry), + }) return remediationInProgressData, false, nil } } @@ -425,6 +524,7 @@ func (r *KubeadmControlPlaneReconciler) canSafelyRemoveEtcdMember(ctx context.Co } // Check member health as reported by machine's health conditions + // TODO: note as phase 2 task if !conditions.IsTrue(machine, controlplanev1.MachineEtcdMemberHealthyCondition) { targetUnhealthyMembers++ unhealthyMembers = append(unhealthyMembers, fmt.Sprintf("%s (%s)", etcdMember, machine.Name)) diff --git a/controlplane/kubeadm/internal/controllers/remediation_test.go b/controlplane/kubeadm/internal/controllers/remediation_test.go index e156b42619e0..83fe0569234b 100644 --- a/controlplane/kubeadm/internal/controllers/remediation_test.go +++ b/controlplane/kubeadm/internal/controllers/remediation_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/cluster-api/controlplane/kubeadm/internal" "sigs.k8s.io/cluster-api/util/collections" "sigs.k8s.io/cluster-api/util/conditions" + v1beta2conditions "sigs.k8s.io/cluster-api/util/conditions/v1beta2" "sigs.k8s.io/cluster-api/util/patch" ) @@ -116,11 +117,13 @@ func TestReconcileUnhealthyMachines(t *testing.T) { if err := env.Get(ctx, client.ObjectKey{Namespace: m.Namespace, Name: m.Name}, m); err != nil { return err } - c := conditions.Get(m, clusterv1.MachineOwnerRemediatedCondition) - if c == nil { - return nil + if c := conditions.Get(m, clusterv1.MachineOwnerRemediatedCondition); c != nil { + return errors.Errorf("condition %s still exists", clusterv1.MachineOwnerRemediatedCondition) } - return errors.Errorf("condition %s still exists", clusterv1.MachineOwnerRemediatedCondition) + if c := v1beta2conditions.Get(m, clusterv1.MachineOwnerRemediatedV1Beta2Condition); c != nil { + return errors.Errorf("condition %s still exists", clusterv1.MachineOwnerRemediatedV1Beta2Condition) + } + return nil }, 10*time.Second).Should(Succeed()) }) @@ -143,9 +146,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { t.Run("reconcileUnhealthyMachines return early if another remediation is in progress", func(t *testing.T) { g := NewWithT(t) - m := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withStuckRemediation()) - conditions.MarkFalse(m, clusterv1.MachineHealthCheckSucceededCondition, clusterv1.MachineHasFailureReason, clusterv1.ConditionSeverityWarning, "") - conditions.MarkFalse(m, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") + m := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) controlPlane := &internal.ControlPlane{ KCP: &controlplanev1.KubeadmControlPlane{ ObjectMeta: metav1.ObjectMeta{ @@ -169,9 +170,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { t.Run("remediation in progress is ignored when stale", func(t *testing.T) { g := NewWithT(t) - m := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withStuckRemediation(), withWaitBeforeDeleteFinalizer()) - conditions.MarkFalse(m, clusterv1.MachineHealthCheckSucceededCondition, clusterv1.MachineHasFailureReason, clusterv1.ConditionSeverityWarning, "") - conditions.MarkFalse(m, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") + m := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) controlPlane := &internal.ControlPlane{ KCP: &controlplanev1.KubeadmControlPlane{ ObjectMeta: metav1.ObjectMeta{ @@ -199,6 +198,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m.Namespace, Name: m.Name}, m) g.Expect(err).ToNot(HaveOccurred()) @@ -211,8 +211,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g := NewWithT(t) m := getDeletingMachine(ns.Name, "m1-unhealthy-deleting-", withMachineHealthCheckFailed()) - conditions.MarkFalse(m, clusterv1.MachineHealthCheckSucceededCondition, clusterv1.MachineHasFailureReason, clusterv1.ConditionSeverityWarning, "") - conditions.MarkFalse(m, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") controlPlane := &internal.ControlPlane{ KCP: &controlplanev1.KubeadmControlPlane{}, Cluster: &clusterv1.Cluster{}, @@ -268,6 +266,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed 3 times (MaxRetry)") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, "KCP can't remediate this machine because the operation already failed 3 times (MaxRetry)") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -323,6 +322,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -381,6 +381,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -433,6 +434,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest 1h0m0s (RetryPeriod)") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason, "KCP can't remediate this machine because the operation already failed in the latest 1h0m0s (RetryPeriod)") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -478,6 +480,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) assertMachineCondition(ctx, g, m, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate if current replicas are less or equal to 1") + assertMachineV1beta2Condition(ctx, g, m, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, "KCP can't remediate if current replicas are less or equal to 1") g.Expect(env.Cleanup(ctx, m)).To(Succeed()) }) @@ -507,6 +510,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine deletion to complete before triggering remediation") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason, "KCP waiting for control plane machine deletion to complete before triggering remediation") g.Expect(env.Cleanup(ctx, m1, m2)).To(Succeed()) }) @@ -536,6 +540,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine provisioning to complete before triggering remediation") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason, "KCP waiting for control plane machine provisioning to complete before triggering remediation") g.Expect(env.Cleanup(ctx, m1, m2)).To(Succeed()) }) @@ -566,6 +571,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine provisioning to complete before triggering remediation") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDeferredV1Beta2Reason, "KCP waiting for control plane machine provisioning to complete before triggering remediation") g.Expect(env.Cleanup(ctx, m1, m2)).To(Succeed()) }) @@ -608,6 +614,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, "KCP can't remediate this machine because this could result in etcd loosing quorum") g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) }) @@ -652,6 +659,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, "KCP can't remediate this machine because this could result in etcd loosing quorum") g.Expect(env.Cleanup(ctx, m1, m2, m3, m4, m5)).To(Succeed()) }) @@ -699,6 +707,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -749,6 +758,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -784,6 +794,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(i - 1)) assertMachineCondition(ctx, g, mi, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, mi, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: mi.Namespace, Name: mi.Name}, mi) g.Expect(err).ToNot(HaveOccurred()) @@ -839,6 +850,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -891,6 +903,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -943,6 +956,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -996,6 +1010,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -1049,6 +1064,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -1097,6 +1113,8 @@ func TestReconcileUnhealthyMachines(t *testing.T) { assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityWarning, "A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to. Skipping remediation") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineCannotBeRemediatedV1Beta2Reason, + "KCP can't remediate this machine because there is no healthy machine to forward etcd leadership to") removeFinalizer(g, m1) g.Expect(env.Cleanup(ctx, m1, m2, m3, m4)).To(Succeed()) @@ -1145,6 +1163,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -1180,6 +1199,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(i - 4)) assertMachineCondition(ctx, g, mi, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, mi, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: mi.Namespace, Name: mi.Name}, mi) g.Expect(err).ToNot(HaveOccurred()) @@ -1250,6 +1270,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m1, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -1285,6 +1306,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(1)) assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m2, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m2.Namespace, Name: m2.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -1360,6 +1382,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m2, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m2.Namespace, Name: m2.Name}, m2) g.Expect(err).ToNot(HaveOccurred()) @@ -1396,6 +1419,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(remediationData.RetryCount).To(Equal(1)) assertMachineCondition(ctx, g, m3, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m3, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") err = env.Get(ctx, client.ObjectKey{Namespace: m3.Namespace, Name: m3.Name}, m3) g.Expect(err).ToNot(HaveOccurred()) @@ -1474,6 +1498,9 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") assertMachineCondition(ctx, g, m3, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") + assertMachineV1beta2Condition(ctx, g, m2, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, controlplanev1.KubeadmControlPlaneMachineRemediationDoneV1Beta2Reason, "") + assertMachineV1beta2Condition(ctx, g, m3, clusterv1.MachineOwnerRemediatedV1Beta2Condition, metav1.ConditionFalse, clusterv1.MachineOwnerRemediatedWaitingForRemediationV1Beta2Reason, "") + err = env.Get(ctx, client.ObjectKey{Namespace: m2.Namespace, Name: m2.Name}, m2) g.Expect(err).ToNot(HaveOccurred()) g.Expect(m2.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) @@ -1900,6 +1927,17 @@ func withMachineHealthCheckFailed() machineOption { return func(machine *clusterv1.Machine) { conditions.MarkFalse(machine, clusterv1.MachineHealthCheckSucceededCondition, clusterv1.MachineHasFailureReason, clusterv1.ConditionSeverityWarning, "") conditions.MarkFalse(machine, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") + + v1beta2conditions.Set(machine, metav1.Condition{ + Type: clusterv1.MachineHealthCheckSucceededV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: clusterv1.MachineHealthCheckNodeDeletedV1Beta2Reason, + }) + v1beta2conditions.Set(machine, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: clusterv1.MachineOwnerRemediatedWaitingForRemediationV1Beta2Reason, + }) } } @@ -1907,6 +1945,17 @@ func withStuckRemediation() machineOption { return func(machine *clusterv1.Machine) { conditions.MarkTrue(machine, clusterv1.MachineHealthCheckSucceededCondition) conditions.MarkFalse(machine, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") + + v1beta2conditions.Set(machine, metav1.Condition{ + Type: clusterv1.MachineHealthCheckSucceededV1Beta2Condition, + Status: metav1.ConditionTrue, + Reason: clusterv1.MachineHealthCheckSucceededV1Beta2Reason, + }) + v1beta2conditions.Set(machine, metav1.Condition{ + Type: clusterv1.MachineOwnerRemediatedV1Beta2Condition, + Status: metav1.ConditionFalse, + Reason: clusterv1.MachineOwnerRemediatedWaitingForRemediationV1Beta2Reason, + }) } } @@ -2031,6 +2080,28 @@ func assertMachineCondition(ctx context.Context, g *WithT, m *clusterv1.Machine, }, 10*time.Second).Should(Succeed()) } +func assertMachineV1beta2Condition(ctx context.Context, g *WithT, m *clusterv1.Machine, t string, status metav1.ConditionStatus, reason string, message string) { + g.Eventually(func() error { + if err := env.Get(ctx, client.ObjectKey{Namespace: m.Namespace, Name: m.Name}, m); err != nil { + return err + } + c := v1beta2conditions.Get(m, t) + if c == nil { + return errors.Errorf("condition %q was nil", t) + } + if c.Status != status { + return errors.Errorf("condition %q status %q did not match %q", t, c.Status, status) + } + if c.Reason != reason { + return errors.Errorf("condition %q reason %q did not match %q", t, c.Reason, reason) + } + if c.Message != message { + return errors.Errorf("condition %q message %q did not match %q", t, c.Message, message) + } + return nil + }, 10*time.Second).Should(Succeed()) +} + func MustMarshalRemediationData(r *RemediationData) string { s, err := r.Marshal() if err != nil {