update switchover

cmd/probe/internal/component/dbmanager.go

@@ -19,8 +19,8 @@ type DBManager interface {
 	AddToCluster()
 	Premote() error
 	Demote() error
-	GetHealthiestMember()
-	HasOtherHealthyLeader() *dcs.Member
+	GetHealthiestMember(*dcs.Cluster, string) *dcs.Member
+	HasOtherHealthyLeader(*dcs.Cluster) *dcs.Member
 	GetCurrentMemberName() string
 	GetMemberAddr(string) string
 	GetLogger() logger.Logger

cmd/probe/internal/component/mongodb/manager.go

@@ -2,6 +2,7 @@ package mongodb
 
 import (
 	"context"
+	"math/rand"
 	"strings"
 	"time"
 
@@ -290,7 +291,61 @@ func (mgr *Manager) Demote() error {
 	return nil
 }
 
-func (mgr *Manager) GetHealthiestMember() {}
-func (mgr *Manager) HasOtherHealthyLeader() *dcs.Member {
+func (mgr *Manager) GetHealthiestMember(cluster *dcs.Cluster, candidate string) *dcs.Member {
+	rsStatus, _ := mgr.GetReplSetStatus(context.TODO())
+	if rsStatus == nil {
+		return nil
+	}
+	healthyMembers := make([]string, 0, len(rsStatus.Members))
+	var leader string
+	for _, member := range rsStatus.Members {
+		if member.Health == 1 {
+			memberName := strings.Split(member.Name, ".")[0]
+			if memberName == candidate {
+				return cluster.GetMemberWithName(candidate)
+			}
+			healthyMembers = append(healthyMembers, memberName)
+			if member.State == 1 {
+				leader = memberName
+			}
+		}
+	}
+
+	if candidate != "" {
+		mgr.Logger.Infof("no health member for candidate: %s", candidate)
+		return nil
+	}
+
+	if leader != "" {
+		return cluster.GetMemberWithName(leader)
+	}
+
+	// TODO: use lag and other info to pick the healthiest member
+	rand.Seed(time.Now().Unix())
+	healthiestMember := healthyMembers[rand.Intn(len(healthyMembers))]
+	return cluster.GetMemberWithName(healthiestMember)
+
+}
+
+func (mgr *Manager) HasOtherHealthyLeader(cluster *dcs.Cluster) *dcs.Member {
+	rsStatus, _ := mgr.GetReplSetStatus(context.TODO())
+	if rsStatus == nil {
+		return nil
+	}
+	var otherLeader string
+	for _, member := range rsStatus.Members {
+		if member.State != 1 {
+			continue
+		}
+		memberName := strings.Split(member.Name, ".")[0]
+		if memberName != mgr.CurrentMemberName {
+			otherLeader = memberName
+		}
+	}
+
+	if otherLeader != "" {
+		return cluster.GetMemberWithName(otherLeader)
+	}
+
 	return nil
 }

cmd/probe/internal/highavailability/ha.go

@@ -57,39 +57,60 @@ func (ha *Ha) RunCycle() {
 	//	dbManager.AddToCluser(cluster)
 	//	cluster.AddThisMember()
 
-	case cluster.IsUnlocked():
-		logger.Infof("Cluster has no leader, attemp to take the leader")
+	case !cluster.IsLocked():
+		ha.logger.Infof("Cluster has no leader, attemp to take the leader")
 		candidate := ""
-		if cluster.SwitchOver != nil && cluster.SwitchOver.Candinate != "" {
-			candiate = cluster.SwitchOver.Candinate
+		if cluster.Switchover != nil && cluster.Switchover.Candidate != "" {
+			candidate = cluster.Switchover.Candidate
 		}
-		healthiestMember := ha.dbManager.GetHealthiesMember(cluster, candinate)
+		healthiestMember := ha.dbManager.GetHealthiestMember(cluster, candidate)
 		if healthiestMember != nil && healthiestMember.Name == ha.dbManager.GetCurrentMemberName() {
-			if dcs.attempAcquireLeader() {
-				dbManager.Premote()
+			if ha.dcs.AttempAcquireLock() == nil {
+				ha.dbManager.Premote()
 			}
 		}
 
-	case cluster.HasLock():
+	case ha.dcs.HasLock():
 		ha.logger.Infof("This member is Cluster's leader")
+		if cluster.Switchover != nil {
+			if cluster.Switchover.Leader == ha.dbManager.GetCurrentMemberName() {
+				ha.dbManager.Demote()
+				ha.dcs.ReleaseLock()
+				break
+			} else if cluster.Switchover.Candidate == "" || cluster.Switchover.Candidate == ha.dbManager.GetCurrentMemberName() {
+				ha.dcs.DeleteSwitchover()
+			}
+		}
+
 		if ok, _ := ha.dbManager.IsLeader(context.TODO()); ok {
 			ha.logger.Infof("Refresh leader ttl")
 			ha.dcs.UpdateLock()
-		} else if ha.dbManager.HasOtherHealthyLeader() != nil {
+		} else if ha.dbManager.HasOtherHealthyLeader(cluster) != nil {
 			ha.logger.Infof("Release leader")
 			ha.dcs.ReleaseLock()
 		} else {
 			ha.dbManager.Premote()
 			ha.dcs.UpdateLock()
 		}
 
-	case cluster.SwitchOver != nil && cluster.SwitchOver.Leader == ha.dbManager.GetCurrentMemberName():
-		logger.Infof("Cluster has no leader, attemp to take the leader")
-		ha.dbManager.Demote()
+	case !ha.dcs.HasLock():
+		if cluster.Switchover != nil {
+			break
+		}
+		if ok, _ := ha.dbManager.IsLeader(context.TODO()); ok {
+			ha.logger.Infof("try to acquire lock")
+			if ha.dcs.AttempAcquireLock() == nil {
+				ha.dbManager.Premote()
+			}
+		}
+
+		// case cluster.SwitchOver != nil && cluster.SwitchOver.Leader == ha.dbManager.GetCurrentMemberName():
+		// 	logger.Infof("Cluster has no leader, attemp to take the leader")
+		// 	ha.dbManager.Demote()
 
-	case cluster.SwitchOver != nil && cluster.SwitchOver.Candidate == ha.dbManager.GetCurrentMemberName():
-		logger.Infof("Cluster has no leader, attemp to take the leader")
-		ha.dbManager.Premote()
+		// case cluster.SwitchOver != nil && cluster.SwitchOver.Candidate == ha.dbManager.GetCurrentMemberName():
+		// 	logger.Infof("Cluster has no leader, attemp to take the leader")
+		// 	ha.dbManager.Premote()
 	}
 }